From d40b2b218a31b9bc3f8198a4860c812cfb1f0aec Mon Sep 17 00:00:00 2001 From: Rashmi Mudduluru Date: Mon, 11 Dec 2023 23:13:50 +0000 Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20change?= =?UTF-8?q?s=20introduced=20through=20rebase?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Created using spr 1.3.4 [skip ci] --- .git-blame-ignore-revs | 3 + .github/CODEOWNERS | 46 +- .github/workflows/llvm-project-tests.yml | 29 +- .github/workflows/spirv-tests.yml | 29 + .mailmap | 2 + bolt/include/bolt/Core/BinaryFunction.h | 15 +- .../include/bolt/Profile/ProfileYAMLMapping.h | 11 + bolt/lib/Core/BinaryFunction.cpp | 10 +- bolt/lib/Core/BinaryFunctionProfile.cpp | 1 + bolt/lib/Core/Exceptions.cpp | 3 +- bolt/lib/Passes/IdenticalCodeFolding.cpp | 6 +- bolt/lib/Passes/LongJmp.cpp | 19 +- bolt/lib/Passes/VeneerElimination.cpp | 1 + bolt/lib/Profile/StaleProfileMatching.cpp | 40 +- bolt/lib/Profile/YAMLProfileReader.cpp | 18 +- bolt/lib/Profile/YAMLProfileWriter.cpp | 1 + bolt/lib/Rewrite/RewriteInstance.cpp | 4 +- .../Inputs/blarge_profile_stale.std-hash.yaml | 56 + .../test/X86/Inputs/blarge_profile_stale.yaml | 1 + bolt/test/X86/reader-stale-yaml-std.test | 68 + clang-tools-extra/clang-doc/Mapper.cpp | 2 +- .../ExpandModularHeadersPPCallbacks.cpp | 2 +- .../bugprone/UnusedReturnValueCheck.cpp | 15 + .../bugprone/UnusedReturnValueCheck.h | 10 +- .../clang-tidy/hicpp/CMakeLists.txt | 1 + .../clang-tidy/hicpp/HICPPTidyModule.cpp | 3 + .../hicpp/IgnoredRemoveResultCheck.cpp | 28 + .../hicpp/IgnoredRemoveResultCheck.h | 29 + .../clang-tidy/misc/IncludeCleanerCheck.cpp | 2 +- .../UnnecessaryCopyInitialization.cpp | 116 +- .../UnnecessaryCopyInitialization.h | 32 +- .../FunctionCognitiveComplexityCheck.cpp | 11 +- clang-tools-extra/clangd/IncludeCleaner.cpp | 8 +- .../clangd/SemanticHighlighting.cpp | 5 +- clang-tools-extra/docs/ReleaseNotes.rst | 6 + .../checks/hicpp/ignored-remove-result.rst | 24 + .../docs/clang-tidy/checks/list.rst | 1 + .../include-cleaner/lib/Analysis.cpp | 6 +- .../modularize/ModuleAssistant.cpp | 2 +- .../pseudo/include/clang-pseudo/Token.h | 5 +- .../checkers/hicpp/ignored-remove-result.cpp | 66 + clang/docs/LanguageExtensions.rst | 25 + clang/docs/ReleaseNotes.rst | 50 +- clang/docs/UsersManual.rst | 19 +- clang/include/clang/AST/Expr.h | 20 +- clang/include/clang/AST/Type.h | 13 +- .../Analyses/UnsafeBufferUsageGadgets.def | 1 + clang/include/clang/Basic/Attr.td | 11 +- clang/include/clang/Basic/AttrDocs.td | 20 +- clang/include/clang/Basic/Builtins.def | 26 + clang/include/clang/Basic/BuiltinsNVPTX.def | 13 +- clang/include/clang/Basic/Cuda.h | 7 +- .../clang/Basic/DiagnosticFrontendKinds.td | 1 + clang/include/clang/Basic/DiagnosticGroups.td | 6 +- .../clang/Basic/DiagnosticParseKinds.td | 2 +- .../clang/Basic/DiagnosticSemaKinds.td | 12 +- clang/include/clang/Basic/DirectoryEntry.h | 74 - clang/include/clang/Basic/FPOptions.def | 1 + clang/include/clang/Basic/Features.def | 3 + clang/include/clang/Basic/FileEntry.h | 66 - clang/include/clang/Basic/LangOptions.def | 2 + clang/include/clang/Basic/LangOptions.h | 4 + clang/include/clang/Basic/Module.h | 4 +- clang/include/clang/Basic/OpenACCKinds.h | 2 +- clang/include/clang/Basic/SourceManager.h | 14 +- clang/include/clang/Basic/SyncScope.h | 69 +- clang/include/clang/Basic/TargetInfo.h | 1 - clang/include/clang/Basic/TargetOSMacros.def | 55 + clang/include/clang/Basic/TokenKinds.def | 5 + clang/include/clang/Basic/arm_sve.td | 60 +- clang/include/clang/Driver/Multilib.h | 2 +- clang/include/clang/Driver/Options.td | 33 + .../include/clang/Frontend/FrontendActions.h | 3 + clang/include/clang/Lex/HeaderSearch.h | 7 +- clang/include/clang/Lex/ModuleMap.h | 14 +- clang/include/clang/Lex/PreprocessorLexer.h | 2 +- clang/include/clang/Lex/PreprocessorOptions.h | 3 + clang/include/clang/Parse/Parser.h | 5 + .../clang/Sema/HLSLExternalSemaSource.h | 4 +- clang/include/clang/Sema/Sema.h | 9 + clang/include/clang/Serialization/ASTReader.h | 7 +- .../include/clang/Serialization/ModuleFile.h | 8 +- clang/lib/AST/Decl.cpp | 1 + clang/lib/AST/Expr.cpp | 26 + clang/lib/AST/ExprConstant.cpp | 34 +- clang/lib/AST/Interp/ByteCodeExprGen.cpp | 1 - clang/lib/AST/Interp/Interp.h | 6 +- clang/lib/AST/Interp/InterpBuiltin.cpp | 52 + clang/lib/AST/Interp/InterpFrame.cpp | 4 +- clang/lib/AST/MicrosoftMangle.cpp | 8 +- clang/lib/AST/RecordLayoutBuilder.cpp | 4 +- clang/lib/AST/StmtPrinter.cpp | 1 + .../lib/Analysis/FlowSensitive/HTMLLogger.cpp | 3 +- clang/lib/Analysis/ThreadSafety.cpp | 26 +- clang/lib/Analysis/UninitializedValues.cpp | 2 +- clang/lib/Analysis/UnsafeBufferUsage.cpp | 99 +- clang/lib/Basic/Cuda.cpp | 5 + clang/lib/Basic/Module.cpp | 2 +- clang/lib/Basic/Targets/AMDGPU.cpp | 80 +- clang/lib/Basic/Targets/AMDGPU.h | 19 +- clang/lib/Basic/Targets/NVPTX.cpp | 3 + clang/lib/Basic/Targets/OSTargets.cpp | 3 + clang/lib/Basic/Targets/RISCV.cpp | 26 +- clang/lib/CodeGen/BackendUtil.cpp | 2 +- clang/lib/CodeGen/CGAtomic.cpp | 125 +- clang/lib/CodeGen/CGBuiltin.cpp | 16 + clang/lib/CodeGen/CGCUDANV.cpp | 27 +- clang/lib/CodeGen/CGCUDARuntime.h | 14 +- clang/lib/CodeGen/CGCall.h | 11 +- clang/lib/CodeGen/CGExprComplex.cpp | 166 +- clang/lib/CodeGen/CGHLSLRuntime.cpp | 9 +- clang/lib/CodeGen/CGHLSLRuntime.h | 2 +- clang/lib/CodeGen/CGOpenMPRuntime.cpp | 9 +- clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp | 1 + clang/lib/CodeGen/CodeGenModule.cpp | 5 +- clang/lib/CodeGen/CodeGenTBAA.cpp | 5 +- clang/lib/CodeGen/Targets/AMDGPU.cpp | 5 + clang/lib/CrossTU/CrossTranslationUnit.cpp | 2 +- .../linux/DirectoryWatcher-linux.cpp | 2 +- clang/lib/Driver/Driver.cpp | 2 +- clang/lib/Driver/Multilib.cpp | 2 +- clang/lib/Driver/ToolChains/Clang.cpp | 74 +- clang/lib/Driver/ToolChains/CommonArgs.cpp | 96 +- clang/lib/Driver/ToolChains/Cuda.cpp | 6 + clang/lib/Driver/ToolChains/Darwin.cpp | 2 +- clang/lib/Driver/ToolChains/Flang.cpp | 14 +- clang/lib/Driver/ToolChains/MSVC.cpp | 4 +- clang/lib/Driver/ToolChains/WebAssembly.cpp | 2 +- .../Serialization/SymbolGraphSerializer.cpp | 3 +- clang/lib/Format/ContinuationIndenter.cpp | 2 +- clang/lib/Format/Format.cpp | 95 +- clang/lib/Format/UnwrappedLineFormatter.cpp | 15 +- clang/lib/Frontend/CompilerInstance.cpp | 2 +- clang/lib/Frontend/CompilerInvocation.cpp | 7 + clang/lib/Frontend/FrontendActions.cpp | 10 + clang/lib/Frontend/InitPreprocessor.cpp | 16 + clang/lib/Lex/HeaderSearch.cpp | 15 +- clang/lib/Lex/InitHeaderSearch.cpp | 3 +- clang/lib/Lex/ModuleMap.cpp | 6 +- clang/lib/Lex/PPDirectives.cpp | 3 +- clang/lib/Lex/Pragma.cpp | 2 +- clang/lib/Lex/PreprocessorLexer.cpp | 3 +- clang/lib/Parse/ParseOpenACC.cpp | 104 +- clang/lib/Parse/ParsePragma.cpp | 40 +- clang/lib/Parse/ParseStmt.cpp | 11 + clang/lib/Parse/Parser.cpp | 3 + clang/lib/Sema/HLSLExternalSemaSource.cpp | 71 +- clang/lib/Sema/Sema.cpp | 2 +- clang/lib/Sema/SemaAttr.cpp | 8 + clang/lib/Sema/SemaChecking.cpp | 61 +- clang/lib/Sema/SemaCodeComplete.cpp | 2 +- clang/lib/Sema/SemaDecl.cpp | 8 +- clang/lib/Sema/SemaDeclAttr.cpp | 25 + clang/lib/Sema/SemaDeclCXX.cpp | 73 +- clang/lib/Sema/SemaInit.cpp | 53 +- clang/lib/Sema/SemaLambda.cpp | 2 +- clang/lib/Sema/SemaRISCVVectorLookup.cpp | 8 +- clang/lib/Sema/SemaStmt.cpp | 3 + clang/lib/Sema/SemaStmtAttr.cpp | 12 + clang/lib/Sema/SemaTemplate.cpp | 127 +- clang/lib/Sema/SemaTemplateInstantiate.cpp | 4 + clang/lib/Sema/SemaType.cpp | 2 +- clang/lib/Serialization/ASTReader.cpp | 10 +- clang/lib/Serialization/ASTReaderDecl.cpp | 21 +- clang/lib/Serialization/ASTReaderStmt.cpp | 106 +- clang/lib/Serialization/ASTWriter.cpp | 8 +- clang/lib/Serialization/ASTWriterDecl.cpp | 156 +- clang/lib/Serialization/ASTWriterStmt.cpp | 36 +- clang/lib/Serialization/GlobalModuleIndex.cpp | 4 +- clang/lib/Serialization/ModuleManager.cpp | 58 +- .../Checkers/BitwiseShiftChecker.cpp | 8 +- .../Checkers/EnumCastOutOfRangeChecker.cpp | 25 +- .../DependencyScanning/ModuleDepCollector.cpp | 2 +- clang/test/AST/HLSL/RWBuffer-AST.hlsl | 4 - clang/test/AST/HLSL/pch.hlsl | 2 +- clang/test/AST/HLSL/pch_with_buf.hlsl | 4 +- .../test/AST/HLSL/resource_binding_attr.hlsl | 12 +- clang/test/AST/Interp/builtin-functions.cpp | 8 + clang/test/AST/Interp/floats.cpp | 4 + clang/test/AST/ms-constexpr.cpp | 28 + .../test/Analysis/enum-cast-out-of-range.cpp | 100 +- clang/test/C/C2x/n2836_n2939.c | 83 + .../test/CXX/class.access/class.friend/p6.cpp | 13 + clang/test/CXX/drs/dr10xx.cpp | 45 +- clang/test/CXX/drs/dr11xx.cpp | 34 +- clang/test/CXX/drs/dr12xx.cpp | 81 +- clang/test/CXX/drs/dr13xx.cpp | 437 ++- clang/test/CXX/drs/dr14xx.cpp | 386 ++- clang/test/CXX/drs/dr15xx.cpp | 408 ++- clang/test/CXX/drs/dr16xx.cpp | 398 ++- clang/test/CXX/drs/dr17xx.cpp | 52 +- clang/test/CXX/drs/dr18xx.cpp | 112 +- clang/test/CXX/drs/dr19xx.cpp | 128 +- clang/test/CXX/drs/dr20xx.cpp | 219 +- clang/test/CXX/drs/dr21xx.cpp | 141 +- clang/test/CXX/drs/dr22xx.cpp | 37 +- clang/test/CXX/drs/dr2354.cpp | 10 - clang/test/CXX/drs/dr2390.cpp | 2 +- clang/test/CXX/drs/dr23xx.cpp | 57 +- clang/test/CXX/drs/dr2406.cpp | 30 - clang/test/CXX/drs/dr24xx.cpp | 51 +- clang/test/CXX/drs/dr25xx.cpp | 108 +- clang/test/CXX/drs/dr26xx.cpp | 133 +- clang/test/CXX/drs/dr27xx.cpp | 25 +- clang/test/CXX/drs/dr412.cpp | 25 +- clang/test/CXX/drs/dr7xx.cpp | 210 +- clang/test/CXX/drs/dr8xx.cpp | 39 +- clang/test/CXX/drs/dr9xx.cpp | 68 +- .../CXX/temp/temp.decls/temp.friend/p1.cpp | 6 +- .../CodeGen/RISCV/riscv-func-attr-target.c | 21 +- .../rvv-intrinsics-handcrafted/rvv-error.c | 2 +- .../acle_sme2_reinterpret_svcount_svbool.c | 47 + .../acle_sme2_vector_qrshr.c | 343 ++ .../acle_sve2p1_create2_bool.c | 38 + .../acle_sve2p1_create4_bool.c | 42 + .../acle_sve2p1_get2_bool.c | 49 + .../acle_sve2p1_get4_bool.c | 72 + .../acle_sve2p1_pext.c | 205 +- .../acle_sve2p1_psel.c | 21 +- .../acle_sve2p1_set2_bool.c | 52 + .../acle_sve2p1_set4_bool.c | 66 + clang/test/CodeGen/arm-target-features.c | 3 + clang/test/CodeGen/complex-math.c | 468 ++- clang/test/CodeGen/cx-complex-range.c | 108 + clang/test/CodeGen/pgo-instrumentation.c | 4 +- clang/test/CodeGen/pragma-cx-limited-range.c | 107 + clang/test/CodeGen/scoped-atomic-ops.c | 331 ++ clang/test/CodeGen/tbaa.c | 116 + clang/test/CodeGenCUDA/offloading-entries.cu | 86 +- .../builtins/RWBuffer-annotations.hlsl | 12 +- .../RasterizerOrderedBuffer-annotations.hlsl | 20 + clang/test/CodeGenHLSL/cbuf.hlsl | 4 +- clang/test/Driver/aarch64-outliner.c | 2 + clang/test/Driver/arm-cortex-cpus-2.c | 5 +- clang/test/Driver/cl-options.c | 4 +- .../test/Driver/clang-offload-bundler-zlib.c | 13 +- .../test/Driver/clang-offload-bundler-zstd.c | 14 +- clang/test/Driver/fdefine-target-os-macros.c | 250 ++ .../test/Driver/hip-offload-compress-zlib.hip | 6 +- .../test/Driver/hip-offload-compress-zstd.hip | 6 +- clang/test/Driver/linker-wrapper-image.c | 76 +- clang/test/Driver/linker-wrapper.c | 4 +- clang/test/Driver/range.c | 39 + clang/test/Driver/rocm-detect.hip | 34 +- clang/test/ExtractAPI/language.c | 101 +- .../Layout/ms-x86-declspec-empty_bases.cpp | 60 + ...a-attribute-supported-attributes-list.test | 1 + clang/test/Misc/target-invalid-cpu-note.c | 4 +- clang/test/Modules/decl-params-determinisim.m | 16 +- .../target_data_use_device_addr_codegen.cpp | 8 +- clang/test/ParserOpenACC/parse-constructs.c | 11 + .../test/ParserOpenACC/parse-wait-construct.c | 157 + clang/test/Preprocessor/init-aarch64.c | 5 + clang/test/Preprocessor/init-loongarch.c | 10 + clang/test/Preprocessor/init.c | 20 + clang/test/Preprocessor/pragma_unknown.c | 2 + .../acle_sve2p1_imm.cpp | 18 + .../Sema/builtin-expect-with-probability.cpp | 1 + clang/test/Sema/riscv-vector-v-check.c | 128 +- clang/test/Sema/scoped-atomic-ops.c | 101 + clang/test/Sema/switch-default.c | 17 + clang/test/SemaCXX/coro-lifetimebound.cpp | 10 +- .../SemaCXX/coro-return-type-and-wrapper.cpp | 9 +- clang/test/SemaCXX/cxx1z-copy-omission.cpp | 27 + clang/test/SemaCXX/friend.cpp | 2 +- clang/test/SemaCXX/ms-constexpr-invalid.cpp | 52 + clang/test/SemaCXX/ms-constexpr-new.cpp | 16 + clang/test/SemaCXX/ms-constexpr.cpp | 37 + .../SemaCXX/warn-thread-safety-analysis.cpp | 8 + ...-unsafe-buffer-usage-fixits-add-assign.cpp | 59 + clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl | 8 + .../test/SemaHLSL/BuiltIns/vector-errors.hlsl | 9 +- clang/test/SemaTemplate/GH71595.cpp | 34 + clang/tools/clang-format/ClangFormat.cpp | 40 +- .../ClangLinkerWrapper.cpp | 2 +- .../clang-linker-wrapper/OffloadWrapper.cpp | 82 +- clang/tools/libclang/CXIndexDataConsumer.cpp | 8 +- clang/tools/libclang/CXIndexDataConsumer.h | 4 +- .../TypeErasedDataflowAnalysisTest.cpp | 78 +- clang/unittests/Basic/FileEntryTest.cpp | 25 - clang/unittests/Format/FormatTest.cpp | 13 + clang/unittests/Format/FormatTestComments.cpp | 15 +- clang/www/c_status.html | 2 +- clang/www/cxx_dr_status.html | 46 +- compiler-rt/lib/asan/asan_mac.cpp | 2 + compiler-rt/lib/builtins/CMakeLists.txt | 11 +- compiler-rt/lib/hwasan/hwasan_flags.inc | 7 + compiler-rt/lib/hwasan/hwasan_linux.cpp | 8 +- .../lib/interception/interception_win.cpp | 1 + compiler-rt/lib/orc/macho_platform.cpp | 143 +- compiler-rt/lib/profile/InstrProfiling.h | 6 + .../lib/profile/InstrProfilingBuffer.c | 4 + compiler-rt/lib/profile/InstrProfilingFile.c | 1 + compiler-rt/lib/scudo/standalone/combined.h | 51 +- .../standalone/fuzz/get_error_info_fuzzer.cpp | 9 +- .../standalone/include/scudo/interface.h | 13 + .../standalone/tests/wrappers_c_test.cpp | 38 + .../lib/scudo/standalone/wrappers_c.inc | 27 +- .../scudo/standalone/wrappers_c_bionic.cpp | 5 +- .../hwasan/TestCases/Linux/fixed-shadow.c | 76 + .../TestCases/Darwin/arm64/Inputs/ret_self.S | 11 + .../TestCases/Darwin/arm64/trivial-dlsym.c | 44 + .../TestCases/Darwin/x86-64/Inputs/ret_self.S | 12 + .../TestCases/Darwin/x86-64/trivial-dlsym.c | 44 + .../reset-default-profile.c | 21 + .../dex/debugger/visualstudio/VisualStudio.py | 7 + flang/include/flang/Runtime/character.h | 11 - flang/include/flang/Runtime/iostat.h | 1 + flang/lib/Frontend/FrontendActions.cpp | 25 +- flang/lib/Lower/OpenMP.cpp | 16 +- flang/lib/Optimizer/Builder/HLFIRTools.cpp | 7 +- flang/lib/Optimizer/CodeGen/CodeGen.cpp | 41 +- flang/lib/Parser/prescan.cpp | 20 +- flang/lib/Parser/prescan.h | 1 + flang/lib/Semantics/check-omp-structure.cpp | 7 +- flang/lib/Semantics/definable.cpp | 23 +- flang/runtime/character.cpp | 142 - flang/runtime/descriptor-io.cpp | 31 +- flang/runtime/descriptor-io.h | 93 +- flang/runtime/edit-input.cpp | 143 +- flang/runtime/edit-output.cpp | 82 +- flang/runtime/internal-unit.cpp | 6 +- flang/runtime/io-stmt.h | 14 +- flang/runtime/iostat.cpp | 2 + flang/runtime/unit.cpp | 65 +- flang/runtime/unit.h | 1 + flang/test/Driver/driver-help-hidden.f90 | 1 + flang/test/Driver/driver-help.f90 | 1 + flang/test/Driver/no-duplicate-main.f90 | 2 + flang/test/Fir/array-coor.fir | 12 +- flang/test/Fir/arrexp.fir | 8 +- flang/test/Fir/convert-to-llvm.fir | 138 +- flang/test/Fir/coordinateof.fir | 8 +- flang/test/Fir/tbaa.fir | 12 +- .../implicit-type-conversion-allocatable.f90 | 26 + flang/test/Lower/OpenACC/acc-set.f90 | 2 +- flang/test/Lower/OpenACC/acc-update.f90 | 2 +- .../PowerPC/ppc-vec-store-elem-order.f90 | 16 +- flang/test/Lower/PowerPC/ppc-vec-store.f90 | 96 +- .../RISCV/riscv-vector-bits-vscale-range.f90 | 21 + .../test/Parser/continuation-before-quote.f90 | 10 - .../Parser/continuation-without-ampersand.f90 | 13 + flang/test/Semantics/OpenMP/target01.f90 | 8 +- flang/test/Semantics/definable01.f90 | 15 + .../unittests/Runtime/NumericalFormatTest.cpp | 29 +- libc/src/__support/FPUtil/FPBits.h | 20 +- libc/src/__support/FPUtil/FloatProperties.h | 241 +- libc/src/__support/FPUtil/Hypot.h | 6 +- .../__support/FPUtil/ManipulationFunctions.h | 12 +- libc/src/__support/FPUtil/NormalFloat.h | 20 +- libc/src/__support/FPUtil/dyadic_float.h | 6 +- libc/src/__support/FPUtil/fpbits_str.h | 2 +- libc/src/__support/FPUtil/generic/FMA.h | 14 +- libc/src/__support/FPUtil/generic/FMod.h | 4 +- libc/src/__support/FPUtil/generic/sqrt.h | 2 +- .../FPUtil/generic/sqrt_80_bit_long_double.h | 4 +- .../__support/FPUtil/x86_64/LongDoubleBits.h | 32 +- .../FPUtil/x86_64/NextAfterLongDouble.h | 12 +- libc/src/__support/UInt.h | 20 +- libc/src/__support/str_to_float.h | 269 +- libc/src/math/generic/expf.cpp | 2 +- libc/src/math/generic/explogxf.h | 7 +- libc/src/math/generic/hypotf.cpp | 4 +- libc/src/math/generic/log.cpp | 2 +- libc/src/math/generic/log10.cpp | 2 +- libc/src/math/generic/log10f.cpp | 2 +- libc/src/math/generic/log1p.cpp | 9 +- libc/src/math/generic/log1pf.cpp | 6 +- libc/src/math/generic/log2.cpp | 2 +- libc/src/math/generic/log2f.cpp | 4 +- libc/src/math/generic/logf.cpp | 4 +- libc/src/math/generic/powf.cpp | 4 +- .../test/src/__support/FPUtil/fpbits_test.cpp | 60 +- libc/test/src/__support/str_to_float_test.cpp | 39 - libc/test/src/math/LdExpTest.h | 2 +- libc/test/src/math/NextAfterTest.h | 14 +- libc/test/src/math/RoundToIntegerTest.h | 4 +- libc/test/src/math/smoke/LdExpTest.h | 2 +- libc/test/src/math/smoke/NextAfterTest.h | 14 +- libc/test/src/math/smoke/NextTowardTest.h | 14 +- .../utils/FPUtil/x86_long_double_test.cpp | 10 +- libc/utils/MPFRWrapper/MPFRUtils.cpp | 8 +- libcxx/docs/ReleaseNotes/18.rst | 1 + libcxx/docs/Status/Cxx2cPapers.csv | 2 +- libcxx/docs/Status/FormatIssues.csv | 2 +- libcxx/include/__format/format_arg_store.h | 6 +- libcxx/include/__format/format_functions.h | 8 +- libcxx/include/__locale | 2 +- libcxx/include/__mdspan/mdspan.h | 11 +- libcxx/include/__random/cauchy_distribution.h | 3 + .../__random/chi_squared_distribution.h | 4 + .../__random/exponential_distribution.h | 3 + .../__random/extreme_value_distribution.h | 3 + .../include/__random/fisher_f_distribution.h | 3 + libcxx/include/__random/gamma_distribution.h | 3 + libcxx/include/__random/is_valid.h | 14 + .../include/__random/lognormal_distribution.h | 4 + libcxx/include/__random/normal_distribution.h | 3 + .../piecewise_constant_distribution.h | 3 + .../__random/piecewise_linear_distribution.h | 3 + .../include/__random/student_t_distribution.h | 3 + .../__random/uniform_real_distribution.h | 3 + .../include/__random/weibull_distribution.h | 4 + libcxx/include/any | 4 +- libcxx/include/format | 4 +- libcxx/include/mdspan | 7 +- libcxx/modules/CMakeLists.txt | 41 +- libcxx/modules/CMakeLists.txt.in | 28 + libcxx/modules/std.compat.cppm.in | 208 ++ libcxx/modules/std.compat/cassert.inc | 12 + libcxx/modules/std.compat/cctype.inc | 25 + libcxx/modules/std.compat/cerrno.inc | 12 + libcxx/modules/std.compat/cfenv.inc | 29 + libcxx/modules/std.compat/cfloat.inc | 12 + libcxx/modules/std.compat/cinttypes.inc | 25 + libcxx/modules/std.compat/climits.inc | 12 + libcxx/modules/std.compat/clocale.inc | 17 + libcxx/modules/std.compat/cmath.inc | 268 ++ libcxx/modules/std.compat/csetjmp.inc | 13 + libcxx/modules/std.compat/csignal.inc | 17 + libcxx/modules/std.compat/cstdarg.inc | 10 + libcxx/modules/std.compat/cstddef.inc | 22 + libcxx/modules/std.compat/cstdint.inc | 50 + libcxx/modules/std.compat/cstdio.inc | 61 + libcxx/modules/std.compat/cstdlib.inc | 72 + libcxx/modules/std.compat/cstring.inc | 36 + libcxx/modules/std.compat/ctime.inc | 28 + libcxx/modules/std.compat/cuchar.inc | 28 + libcxx/modules/std.compat/cwchar.inc | 80 + libcxx/modules/std.compat/cwctype.inc | 35 + libcxx/modules/std.cppm.in | 46 +- .../atomics.types.float/lockfree.pass.cpp | 3 +- libcxx/test/libcxx/module_std.gen.py | 246 +- libcxx/test/libcxx/module_std_compat.gen.py | 62 + .../rand.req.urng/valid_real_type.verify.cpp | 109 + .../libcxx/selftest/stdin-is-piped.sh.cpp | 2 - libcxx/test/lit.local.cfg | 14 + .../alg.unique/ranges_unique_copy.pass.cpp | 4 +- .../sort.heap/ranges_sort_heap.pass.cpp | 2 +- .../atomics.types.float/assign.pass.cpp | 3 +- .../compare_exchange_strong.pass.cpp | 2 +- .../compare_exchange_weak.pass.cpp | 8 +- .../atomics.types.float/ctor.pass.cpp | 3 +- .../atomics.types.float/exchange.pass.cpp | 6 +- .../atomics.types.float/fetch_add.pass.cpp | 10 +- .../atomics.types.float/fetch_sub.pass.cpp | 10 +- .../atomics.types.float/load.pass.cpp | 5 +- .../atomics.types.float/lockfree.pass.cpp | 3 +- .../atomics.types.float/notify_all.pass.cpp | 3 +- .../atomics.types.float/notify_one.pass.cpp | 3 +- .../operator.float.pass.cpp | 3 +- .../operator.minus_equals.pass.cpp | 7 +- .../operator.plus_equals.pass.cpp | 7 +- .../atomics.types.float/store.pass.cpp | 5 +- .../atomics.types.float/wait.pass.cpp | 6 +- .../insert_range_sequence_containers.h | 3 +- .../unord.map/eq.different_hash.pass.cpp | 4 +- .../unord.multimap/eq.different_hash.pass.cpp | 4 +- .../unord.multiset/eq.different_hash.pass.cpp | 4 +- .../unord.set/eq.different_hash.pass.cpp | 8 +- .../layout_stride/index_operator.pass.cpp | 10 +- .../mdspan/mdspan/ctor.dh_array.pass.cpp | 2 +- .../mdspan/mdspan/ctor.dh_extents.pass.cpp | 2 +- .../views/mdspan/mdspan/ctor.dh_map.pass.cpp | 2 +- .../mdspan/mdspan/ctor.dh_map_acc.pass.cpp | 2 +- .../views/mdspan/mdspan/ctor.dh_span.pass.cpp | 2 +- .../views/mdspan/mdspan/properties.pass.cpp | 28 +- .../ifstream.members/buffered_reads.pass.cpp | 4 +- .../ofstream.members/buffered_writes.pass.cpp | 4 +- .../path.member/path.assign/source.pass.cpp | 2 +- .../fs.enum/enum.copy_options.pass.cpp | 2 +- .../fs.enum/enum.directory_options.pass.cpp | 2 +- .../fs.enum/enum.file_type.pass.cpp | 2 +- .../fs.enum/enum.path.format.pass.cpp | 4 +- .../fs.enum/enum.perm_options.pass.cpp | 2 +- .../filesystems/fs.enum/enum.perms.pass.cpp | 2 +- .../fs.op.absolute/absolute.pass.cpp | 2 +- .../fs.op.canonical/canonical.pass.cpp | 4 +- .../fs.op.permissions/permissions.pass.cpp | 2 +- .../temp_directory_path.pass.cpp | 4 +- .../print.fun/no_file_description.pass.cpp | 6 +- .../print.fun/vprint_nonunicode.sh.cpp | 8 +- .../print.fun/vprint_unicode.sh.cpp | 8 +- .../narrow.stream.objects/cin.sh.cpp | 6 +- .../member_swap_noexcept.pass.cpp | 2 +- .../nonmember_swap_noexcept.pass.cpp | 2 +- .../syncstream.syncbuf.assign/swap.pass.cpp | 3 +- .../iterator.cust.move/iter_move.pass.cpp | 6 +- .../new.size.replace.indirect.pass.cpp | 4 +- .../new.size.replace.pass.cpp | 4 +- ...new.size_nothrow.replace.indirect.pass.cpp | 4 +- .../new.size.replace.pass.cpp | 4 +- .../new.size_align_nothrow.pass.cpp | 3 +- .../new.size_nothrow.pass.cpp | 3 +- ...new.size_nothrow.replace.indirect.pass.cpp | 4 +- libcxx/test/std/modules/std.compat.pass.cpp | 18 + .../range.elements/general.pass.cpp | 2 +- .../range.elements/iterator/base.pass.cpp | 2 +- .../range.elements/iterator/deref.pass.cpp | 4 +- .../iterator/member_types.compile.pass.cpp | 4 +- .../range.utility.conv/container.h | 6 +- .../thread.thread.constr/F.pass.cpp | 4 +- .../test/std/time/time.syn/formatter_tests.h | 48 +- .../make_format_args.pass.cpp | 15 +- .../format.arg.store/make_format_args.sh.cpp | 3 +- .../make_wformat_args.pass.cpp | 15 +- .../format.args/ctad.compile.pass.cpp | 5 +- .../format.args/ctor.pass.cpp | 9 +- .../format.context/arg.pass.cpp | 6 +- .../format.context/ctor.pass.cpp | 5 +- .../format.context/locale.pass.cpp | 5 +- .../formatter.string.pass.cpp | 3 +- .../escaped_output.unicode.pass.cpp | 12 +- .../refwrap.const/type_conv_ctor.pass.cpp | 20 +- .../allocate_shared_for_overwrite.pass.cpp | 2 +- .../make_shared_for_overwrite.pass.cpp | 5 +- ...unique_for_overwrite.default_init.pass.cpp | 5 +- libcxx/test/support/allocators.h | 2 +- libcxx/test/support/concat_macros.h | 149 +- .../test/support/msvc_stdlib_force_include.h | 2 + .../header_exportable_declarations.cpp | 134 +- .../header_exportable_declarations.hpp | 34 +- libcxx/utils/CMakeLists.txt | 13 +- ..._cppm_in.py => generate_libcxx_cppm_in.py} | 71 +- libcxx/utils/libcxx/test/features.py | 24 +- libcxx/utils/libcxx/test/modules.py | 302 ++ libcxxabi/src/cxa_personality.cpp | 7 +- .../test/native/x86_64/lpstart-zero.pass.sh.s | 109 + lld/COFF/PDB.cpp | 2 +- lld/ELF/DWARF.h | 1 + lld/ELF/InputFiles.cpp | 1 + lld/ELF/InputSection.cpp | 41 +- lld/MachO/InputFiles.cpp | 2 +- lld/test/ELF/debug-dead-reloc-32.s | 11 + lld/test/ELF/debug-dead-reloc.s | 11 +- lld/test/ELF/emulation-amdgpu.s | 17 +- lld/test/ELF/i386-debug-noabs.test | 3 +- lld/test/ELF/non-abs-reloc.s | 3 +- lld/test/ELF/x86-64-dyn-rel-error.s | 52 +- lld/test/ELF/x86-64-dyn-rel-error2.s | 14 - lld/test/ELF/x86-64-reloc-32.s | 3 - lldb/include/lldb/Core/Address.h | 12 +- lldb/include/lldb/Core/Debugger.h | 4 + lldb/include/lldb/Symbol/Symbol.h | 4 +- lldb/include/lldb/Symbol/SymbolContext.h | 14 +- lldb/include/lldb/Target/Process.h | 10 +- lldb/include/lldb/Utility/Stream.h | 34 + lldb/source/Commands/CommandCompletions.cpp | 2 +- lldb/source/Commands/CommandObjectTarget.cpp | 20 +- lldb/source/Core/Address.cpp | 32 +- lldb/source/Core/CoreProperties.td | 8 + lldb/source/Core/Debugger.cpp | 14 +- .../Process/MacOSX-Kernel/ProcessKDP.cpp | 2 +- .../Process/elf-core/ProcessElfCore.cpp | 2 +- .../Process/gdb-remote/ProcessGDBRemote.cpp | 2 +- .../Process/mach-core/ProcessMachCore.cpp | 2 +- .../Process/minidump/ProcessMinidump.cpp | 2 +- .../Process/scripted/ScriptedProcess.cpp | 2 +- .../SymbolFile/DWARF/DWARFDeclContext.cpp | 41 +- .../SymbolFile/DWARF/SymbolFileDWARF.cpp | 60 +- lldb/source/Symbol/Symbol.cpp | 27 +- lldb/source/Symbol/SymbolContext.cpp | 31 +- lldb/source/Symbol/Variable.cpp | 3 +- lldb/source/Target/Process.cpp | 14 +- lldb/source/Target/ProcessTrace.cpp | 2 +- lldb/source/Target/Target.cpp | 2 +- lldb/source/Utility/Stream.cpp | 30 + lldb/test/API/driver/quit_speed/Makefile | 3 + .../driver/quit_speed/TestQuitWithProcess.py | 34 + lldb/test/API/driver/quit_speed/main.c | 8 + .../location-list-lookup/Makefile | 2 +- .../TestLocationListLookup.py | 38 +- .../location-list-lookup/main.c | 23 - .../location-list-lookup/main.cpp | 23 + .../TestConstStaticIntegralMember.py | 8 +- .../lldb-dap/variables/TestDAP_variables.py | 10 +- .../Commands/command-image-lookup-color.test | 60 + lldb/tools/lldb-dap/JSONUtils.cpp | 47 +- llvm/CMakeLists.txt | 6 +- llvm/cmake/modules/HandleLLVMOptions.cmake | 3 + llvm/docs/CoverageMappingFormat.rst | 2 +- llvm/docs/GettingStarted.rst | 2 +- llvm/docs/LangRef.rst | 5 + llvm/docs/MIRLangRef.rst | 4 +- llvm/docs/ReleaseNotes.rst | 26 +- .../LLJITWithRemoteDebugging.cpp | 16 +- llvm/include/llvm-c/Core.h | 95 + llvm/include/llvm-c/LLJIT.h | 2 +- llvm/include/llvm-c/LLJITUtils.h | 52 + llvm/include/llvm-c/Types.h | 5 + llvm/include/llvm/ADT/SmallString.h | 12 +- llvm/include/llvm/ADT/SparseBitVector.h | 2 +- llvm/include/llvm/ADT/StringRef.h | 12 - .../Analysis/ScalarEvolutionExpressions.h | 1 - .../llvm/Analysis/TargetTransformInfo.h | 6 +- llvm/include/llvm/AsmParser/LLParser.h | 31 +- llvm/include/llvm/AsmParser/LLToken.h | 1 + .../include/llvm/BinaryFormat/DynamicTags.def | 6 + llvm/include/llvm/BinaryFormat/ELF.h | 8 + .../llvm/BinaryFormat/ELFRelocs/AArch64.def | 3 +- llvm/include/llvm/CodeGen/CallingConvLower.h | 1 + .../include/llvm/CodeGen/CodeGenPassBuilder.h | 6 +- .../CodeGen/GlobalISel/LegacyLegalizerInfo.h | 1 + .../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 1 + .../CodeGen/GlobalISel/MachineIRBuilder.h | 7 +- llvm/include/llvm/CodeGen/InterleavedAccess.h | 34 + .../llvm/CodeGen/MachinePassRegistry.def | 4 +- llvm/include/llvm/CodeGen/SchedulerRegistry.h | 2 +- llvm/include/llvm/CodeGen/TargetSchedule.h | 2 +- .../llvm/DebugInfo/CodeView/CodeView.h | 11 +- .../llvm/DebugInfo/PDB/Native/FormatUtil.h | 4 +- .../llvm/ExecutionEngine/JITLink/aarch64.h | 2 +- .../llvm/ExecutionEngine/Orc/MachOPlatform.h | 65 +- .../include/llvm/Frontend/HLSL/HLSLResource.h | 7 +- .../llvm/Frontend/Offloading/Utility.h | 31 +- .../include/llvm/IR/DebugProgramInstruction.h | 2 +- llvm/include/llvm/IR/Dominators.h | 2 - .../llvm/IR/GetElementPtrTypeIterator.h | 1 - llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 3 +- llvm/include/llvm/IR/OperandTraits.h | 4 +- llvm/include/llvm/IR/PatternMatch.h | 8 + llvm/include/llvm/MC/MCAsmBackend.h | 6 +- llvm/include/llvm/MC/MCAssembler.h | 9 +- llvm/include/llvm/MC/MCInstrItineraries.h | 4 +- llvm/include/llvm/MC/MCSchedule.h | 1 - llvm/include/llvm/MC/MCSymbolXCOFF.h | 7 + llvm/include/llvm/Object/XCOFFObjectFile.h | 2 +- llvm/include/llvm/Support/AMDGPUAddrSpace.h | 83 + llvm/include/llvm/Support/LLVMDriver.h | 2 - llvm/include/llvm/Support/TargetOpcodes.def | 5 +- llvm/include/llvm/Support/TypeSize.h | 1 - llvm/include/llvm/Target/GenericOpcodes.td | 9 + .../llvm/TargetParser/AArch64TargetParser.h | 5 +- .../llvm/TargetParser/ARMTargetParser.def | 5 +- llvm/include/llvm/TextAPI/RecordsSlice.h | 9 +- .../llvm/Transforms/HipStdPar/HipStdPar.h | 1 - .../llvm/Transforms/IPO/BlockExtractor.h | 1 - .../llvm/Transforms/IPO/EmbedBitcodePass.h | 1 - .../include/llvm/Transforms/Instrumentation.h | 6 + .../Instrumentation/InstrProfiling.h | 46 +- .../Transforms/Instrumentation/MemProfiler.h | 2 - .../llvm/Transforms/Scalar/Reassociate.h | 3 +- .../llvm/Transforms/Scalar/Scalarizer.h | 1 - .../Transforms/Scalar/SimpleLoopUnswitch.h | 1 - .../llvm/Transforms/Utils/ValueMapper.h | 1 - llvm/lib/Analysis/AliasSetTracker.cpp | 10 +- llvm/lib/Analysis/DomConditionCache.cpp | 3 +- llvm/lib/Analysis/IVDescriptors.cpp | 2 - llvm/lib/Analysis/ValueTracking.cpp | 22 +- llvm/lib/AsmParser/LLLexer.cpp | 1 + llvm/lib/AsmParser/LLParser.cpp | 66 +- llvm/lib/Bitcode/Reader/BitcodeReader.cpp | 22 +- llvm/lib/Bitcode/Reader/ValueList.cpp | 1 - .../CodeGen/AssignmentTrackingAnalysis.cpp | 27 +- llvm/lib/CodeGen/DwarfEHPrepare.cpp | 2 +- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 15 + .../CodeGen/GlobalISel/LegalizerHelper.cpp | 52 + .../CodeGen/GlobalISel/MachineIRBuilder.cpp | 12 + llvm/lib/CodeGen/InterleavedAccessPass.cpp | 98 +- llvm/lib/CodeGen/MachineScheduler.cpp | 12 +- llvm/lib/CodeGen/MachineVerifier.cpp | 23 + llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 17 +- llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 3 +- .../SelectionDAG/LegalizeFloatTypes.cpp | 20 + llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 1 + .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 6 + .../SelectionDAG/SelectionDAGBuilder.cpp | 2 +- .../CodeGen/TargetLoweringObjectFileImpl.cpp | 18 +- llvm/lib/CodeGen/WinEHPrepare.cpp | 4 +- llvm/lib/DebugInfo/DWARF/DWARFDie.cpp | 34 +- llvm/lib/DebugInfo/PDB/Native/FormatUtil.cpp | 5 +- .../Orc/Debugging/CMakeLists.txt | 1 + .../Orc/Debugging/DebuggerSupport.cpp | 2 +- .../Orc/Debugging/LLJITUtilsCBindings.cpp | 22 + .../lib/ExecutionEngine/Orc/MachOPlatform.cpp | 261 +- llvm/lib/Frontend/HLSL/HLSLResource.cpp | 19 +- llvm/lib/Frontend/Offloading/Utility.cpp | 4 +- llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp | 2 +- llvm/lib/IR/AsmWriter.cpp | 12 +- llvm/lib/IR/AutoUpgrade.cpp | 3 +- llvm/lib/IR/Core.cpp | 69 + llvm/lib/IR/DebugProgramInstruction.cpp | 5 +- llvm/lib/IR/Type.cpp | 13 +- llvm/lib/IR/Verifier.cpp | 2 +- llvm/lib/MC/MCAssembler.cpp | 22 +- llvm/lib/MC/MCELFStreamer.cpp | 1 - llvm/lib/MC/MCObjectStreamer.cpp | 5 +- llvm/lib/MC/MCSchedule.cpp | 6 +- llvm/lib/MC/MCSubtargetInfo.cpp | 4 +- llvm/lib/ObjCopy/CommonConfig.cpp | 7 + llvm/lib/Object/ELF.cpp | 1 + llvm/lib/ObjectYAML/ELFYAML.cpp | 1 + llvm/lib/ObjectYAML/XCOFFYAML.cpp | 36 +- llvm/lib/Passes/PassBuilder.cpp | 1 + llvm/lib/Passes/PassBuilderPipelines.cpp | 4 +- llvm/lib/Passes/PassRegistry.def | 5 +- llvm/lib/Support/CommandLine.cpp | 2 +- llvm/lib/Support/Windows/Path.inc | 4 +- llvm/lib/TableGen/JSONBackend.cpp | 1 - llvm/lib/Target/AArch64/AArch64.td | 4 +- llvm/lib/Target/AArch64/AArch64CollectLOH.cpp | 1 - .../AArch64/AArch64ExpandPseudoInsts.cpp | 1 - .../Target/AArch64/AArch64FalkorHWPFFix.cpp | 1 - .../Target/AArch64/AArch64ISelLowering.cpp | 33 +- llvm/lib/Target/AArch64/AArch64InstrGISel.td | 4 +- llvm/lib/Target/AArch64/AArch64InstrInfo.cpp | 6 +- .../Target/AArch64/AArch64SLSHardening.cpp | 1 - .../AArch64/AArch64TargetTransformInfo.h | 1 + .../AArch64/GISel/AArch64LegalizerInfo.cpp | 55 +- .../AArch64/GISel/AArch64LegalizerInfo.h | 1 + .../MCTargetDesc/AArch64AsmBackend.cpp | 6 +- llvm/lib/Target/AArch64/SMEABIPass.cpp | 1 - llvm/lib/Target/AMDGPU/AMDGPU.h | 68 +- .../lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h | 1 - .../Target/AMDGPU/AMDGPUHSAMetadataStreamer.h | 1 - llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp | 49 +- llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h | 2 +- llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 9 +- .../lib/Target/AMDGPU/AMDGPUMachineFunction.h | 1 - .../Disassembler/AMDGPUDisassembler.cpp | 8 +- llvm/lib/Target/AMDGPU/EXPInstructions.td | 26 +- llvm/lib/Target/AMDGPU/GCNRegPressure.cpp | 50 +- llvm/lib/Target/AMDGPU/GCNRegPressure.h | 26 + llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp | 6 +- .../AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp | 6 +- llvm/lib/Target/AMDGPU/SIDefines.h | 5 +- .../Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp | 7 +- .../Target/AMDGPU/Utils/AMDGPUMemoryUtils.h | 2 - llvm/lib/Target/AMDGPU/VINTERPInstructions.td | 38 +- llvm/lib/Target/AMDGPU/VOP3Instructions.td | 15 +- llvm/lib/Target/ARM/ARM.td | 13 +- llvm/lib/Target/ARM/ARMSubtarget.cpp | 1 + llvm/lib/Target/ARM/ARMSubtarget.h | 2 + .../Target/ARM/MCTargetDesc/ARMAsmBackend.cpp | 3 +- .../Target/ARM/MCTargetDesc/ARMAsmBackend.h | 3 +- .../ARM/MCTargetDesc/ARMUnwindOpAsm.cpp | 2 +- .../MCTargetDesc/ARMWinCOFFObjectWriter.cpp | 1 - .../Target/AVR/MCTargetDesc/AVRAsmBackend.cpp | 3 +- .../Target/AVR/MCTargetDesc/AVRAsmBackend.h | 3 +- llvm/lib/Target/BPF/BPF.h | 7 + llvm/lib/Target/BPF/BPF.td | 1 + llvm/lib/Target/BPF/BPFInstrInfo.td | 4 +- llvm/lib/Target/BPF/BPFSubtarget.cpp | 32 +- llvm/lib/Target/BPF/BPFSubtarget.h | 21 +- llvm/lib/Target/BPF/BPFTargetMachine.cpp | 31 + llvm/lib/Target/BPF/CMakeLists.txt | 7 + llvm/lib/Target/BPF/GISel/BPFCallLowering.cpp | 46 + llvm/lib/Target/BPF/GISel/BPFCallLowering.h | 39 + .../BPF/GISel/BPFInstructionSelector.cpp | 91 + .../lib/Target/BPF/GISel/BPFLegalizerInfo.cpp | 22 + llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.h | 28 + .../Target/BPF/GISel/BPFRegisterBankInfo.cpp | 25 + .../Target/BPF/GISel/BPFRegisterBankInfo.h | 39 + llvm/lib/Target/BPF/GISel/BPFRegisterBanks.td | 15 + .../BPF/MCTargetDesc/BPFInstPrinter.cpp | 5 +- .../Target/BPF/MCTargetDesc/BPFMCTargetDesc.h | 5 +- .../CSKY/MCTargetDesc/CSKYAsmBackend.cpp | 3 +- .../Target/CSKY/MCTargetDesc/CSKYAsmBackend.h | 3 +- llvm/lib/Target/DirectX/DXILResource.cpp | 7 +- .../lib/Target/Hexagon/HexagonTargetMachine.h | 2 - .../MCTargetDesc/HexagonAsmBackend.cpp | 3 +- .../Hexagon/MCTargetDesc/HexagonMCChecker.h | 1 - .../LoongArch/LoongArchISelLowering.cpp | 18 + .../Target/LoongArch/LoongArchISelLowering.h | 1 - .../LoongArch/LoongArchLASXInstrInfo.td | 147 + .../Target/LoongArch/LoongArchLSXInstrInfo.td | 170 + .../MCTargetDesc/LoongArchAsmBackend.cpp | 5 +- .../MCTargetDesc/LoongArchAsmBackend.h | 3 +- .../Mips/MCTargetDesc/MipsAsmBackend.cpp | 3 +- .../Target/Mips/MCTargetDesc/MipsAsmBackend.h | 3 +- llvm/lib/Target/Mips/Mips16FrameLowering.cpp | 1 - llvm/lib/Target/Mips/MipsISelLowering.h | 1 - llvm/lib/Target/Mips/MipsMCInstLower.cpp | 1 - llvm/lib/Target/Mips/MipsSEISelLowering.cpp | 1 - llvm/lib/Target/NVPTX/NVPTX.td | 19 +- llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 48 + llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 3 + llvm/lib/Target/NVPTX/NVPTXInstrInfo.td | 2 +- llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp | 7 +- llvm/lib/Target/NVPTX/NVPTXSubtarget.h | 17 +- .../Target/PowerPC/AsmParser/PPCAsmParser.cpp | 29 + .../PowerPC/MCTargetDesc/PPCAsmBackend.cpp | 3 +- .../PowerPC/MCTargetDesc/PPCELFStreamer.h | 1 - .../PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp | 1 - .../PowerPC/MCTargetDesc/PPCPredicates.cpp | 1 - llvm/lib/Target/PowerPC/P10InstrResources.td | 3 +- llvm/lib/Target/PowerPC/PPC.h | 72 +- llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp | 31 +- .../lib/Target/PowerPC/PPCBack2BackFusion.def | 1 + llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 16 +- llvm/lib/Target/PowerPC/PPCInstrFormats.td | 32 + llvm/lib/Target/PowerPC/PPCInstrInfo.cpp | 37 +- llvm/lib/Target/PowerPC/PPCInstrInfo.h | 30 +- llvm/lib/Target/PowerPC/PPCInstrInfo.td | 34 +- llvm/lib/Target/PowerPC/PPCInstrP10.td | 207 +- llvm/lib/Target/PowerPC/PPCMCInstLower.cpp | 19 +- .../lib/Target/PowerPC/PPCPreEmitPeephole.cpp | 2 +- llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp | 2 +- .../Target/PowerPC/PPCTargetTransformInfo.cpp | 1 - .../Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 54 +- .../Target/RISCV/GISel/RISCVLegalizerInfo.h | 3 + .../RISCV/MCTargetDesc/RISCVAsmBackend.cpp | 11 +- .../RISCV/MCTargetDesc/RISCVAsmBackend.h | 7 +- llvm/lib/Target/RISCV/RISCV.h | 5 - .../RISCV/RISCVDeadRegisterDefinitions.cpp | 5 +- .../Target/RISCV/RISCVExpandPseudoInsts.cpp | 8 +- llvm/lib/Target/RISCV/RISCVFeatures.td | 19 +- llvm/lib/Target/RISCV/RISCVFoldMasks.cpp | 4 +- .../Target/RISCV/RISCVInsertReadWriteCSR.cpp | 4 +- llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp | 4 +- .../lib/Target/RISCV/RISCVInsertWriteVXRM.cpp | 4 +- .../Target/RISCV/RISCVInstrInfoVPseudos.td | 6 +- llvm/lib/Target/RISCV/RISCVMacroFusion.cpp | 121 +- .../Target/RISCV/RISCVMakeCompressible.cpp | 4 +- .../lib/Target/RISCV/RISCVMergeBaseOffset.cpp | 1 - llvm/lib/Target/RISCV/RISCVMoveMerger.cpp | 4 +- llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp | 16 +- llvm/lib/Target/RISCV/RISCVProcessors.td | 2 +- .../Target/RISCV/RISCVPushPopOptimizer.cpp | 4 +- llvm/lib/Target/RISCV/RISCVRVVInitUndef.cpp | 4 +- llvm/lib/Target/RISCV/RISCVSubtarget.h | 6 +- llvm/lib/Target/RISCV/RISCVTargetMachine.cpp | 11 +- llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp | 1 - llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp | 7 +- llvm/lib/Target/SPIRV/SPIRVUtils.cpp | 7 +- .../Sparc/MCTargetDesc/SparcAsmBackend.cpp | 3 +- .../MCTargetDesc/SystemZMCAsmBackend.cpp | 8 +- .../Target/SystemZ/SystemZISelDAGToDAG.cpp | 3 + .../Target/SystemZ/SystemZISelLowering.cpp | 408 +-- llvm/lib/Target/SystemZ/SystemZISelLowering.h | 5 +- .../lib/Target/SystemZ/SystemZInstrFormats.td | 24 - llvm/lib/Target/SystemZ/SystemZInstrInfo.td | 87 +- .../Target/VE/MCTargetDesc/VEAsmBackend.cpp | 3 +- llvm/lib/Target/X86/MCA/X86CustomBehaviour.h | 1 - .../Target/X86/MCTargetDesc/X86AsmBackend.cpp | 7 +- .../X86/MCTargetDesc/X86InstComments.cpp | 4 +- .../Target/X86/X86FixupVectorConstants.cpp | 21 +- llvm/lib/Target/X86/X86FlagsCopyLowering.cpp | 1 - llvm/lib/Target/X86/X86ISelLowering.cpp | 106 +- llvm/lib/Target/X86/X86InstrInfo.cpp | 31 +- llvm/lib/Target/X86/X86InstrSSE.td | 30 +- llvm/lib/Target/X86/X86InstrSystem.td | 4 + llvm/lib/Target/X86/X86InstrUtils.td | 1 + llvm/lib/Target/X86/X86InstrVMX.td | 8 +- llvm/lib/Target/X86/X86MCInstLower.cpp | 8 +- llvm/lib/Target/X86/X86ReplaceableInstrs.def | 2 +- llvm/lib/Target/X86/X86SchedAlderlakeP.td | 2 +- llvm/lib/Target/X86/X86SchedBroadwell.td | 4 +- llvm/lib/Target/X86/X86SchedHaswell.td | 4 +- llvm/lib/Target/X86/X86SchedIceLake.td | 4 +- llvm/lib/Target/X86/X86SchedSapphireRapids.td | 2 +- llvm/lib/Target/X86/X86SchedSkylakeClient.td | 4 +- llvm/lib/Target/X86/X86SchedSkylakeServer.td | 4 +- llvm/lib/Target/X86/X86ScheduleBdVer2.td | 2 +- llvm/lib/Target/X86/X86ScheduleBtVer2.td | 2 +- llvm/lib/Target/X86/X86ScheduleZnver1.td | 4 +- llvm/lib/Target/X86/X86ScheduleZnver2.td | 4 +- .../X86/X86SpeculativeLoadHardening.cpp | 1 - llvm/lib/TargetParser/Host.cpp | 1 + llvm/lib/Transforms/Hello/CMakeLists.txt | 2 +- .../Transforms/IPO/AttributorAttributes.cpp | 41 +- .../InstCombine/InstCombineAndOrXor.cpp | 8 +- .../InstCombine/InstCombineCalls.cpp | 34 + .../InstCombine/InstCombineCompares.cpp | 45 +- .../InstCombineLoadStoreAlloca.cpp | 23 +- .../InstCombine/InstCombineMulDivRem.cpp | 3 +- .../InstCombine/InstCombineNegator.cpp | 3 - .../InstCombine/InstructionCombining.cpp | 1 + .../Instrumentation/AddressSanitizer.cpp | 3 + .../Instrumentation/InstrProfiling.cpp | 130 +- .../Instrumentation/Instrumentation.cpp | 7 + .../Instrumentation/SanitizerCoverage.cpp | 6 +- .../Scalar/AlignmentFromAssumptions.cpp | 22 +- .../Scalar/ConstraintElimination.cpp | 2 - .../Scalar/InductiveRangeCheckElimination.cpp | 1 - llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp | 8 +- .../lib/Transforms/Scalar/MemCpyOptimizer.cpp | 7 +- llvm/lib/Transforms/Scalar/Reassociate.cpp | 31 +- llvm/lib/Transforms/Scalar/SCCP.cpp | 1 - llvm/lib/Transforms/Utils/LowerSwitch.cpp | 1 - .../Utils/ScalarEvolutionExpander.cpp | 20 +- llvm/lib/Transforms/Utils/SimplifyCFG.cpp | 1 - .../Vectorize/LoadStoreVectorizer.cpp | 1 - .../Transforms/Vectorize/LoopVectorize.cpp | 26 +- .../Transforms/Vectorize/SLPVectorizer.cpp | 148 +- llvm/lib/Transforms/Vectorize/VPlan.cpp | 22 +- llvm/lib/Transforms/Vectorize/VPlan.h | 36 +- llvm/lib/Transforms/Vectorize/VPlanAnalysis.h | 3 - .../lib/Transforms/Vectorize/VPlanRecipes.cpp | 28 +- .../Transforms/Vectorize/VPlanTransforms.h | 2 - llvm/lib/Transforms/Vectorize/VPlanValue.h | 21 +- .../depend_diff_types_opaque_ptr.ll | 179 - .../num-iters-for-store-load-conflict.ll | 310 ++ .../Analysis/ScalarEvolution/trip-count.ll | 25 + .../knownbits-and-or-xor-lowbit.ll | 7 +- .../Assembler/constant-splat-diagnostics.ll | 40 + llvm/test/Assembler/constant-splat.ll | 67 + llvm/test/Assembler/named-metadata.ll | 4 + .../Assembler/struct-ret-without-upgrade.ll | 11 + llvm/test/Assembler/summary-parsing-error.ll | 4 + llvm/test/Bindings/llvm-c/echo.ll | 11 + .../AArch64/GlobalISel/arm64-irtranslator.ll | 165 +- .../GlobalISel/legalizer-info-validation.mir | 3 + .../aarch64-neon-vector-insert-uaddlv.ll | 3 +- ...rm64-promote-const-complex-initializers.ll | 20 +- llvm/test/CodeGen/AArch64/atomic-oversize.ll | 11 + llvm/test/CodeGen/AArch64/cmp-select-sign.ll | 14 +- llvm/test/CodeGen/AArch64/dag-numsignbits.ll | 5 +- llvm/test/CodeGen/AArch64/fpenv.ll | 89 +- .../CodeGen/AArch64/misched-fusion-aes.ll | 1 - .../AArch64/neon-compare-instructions.ll | 598 +++- llvm/test/CodeGen/AArch64/signbit-shift.ll | 24 +- .../test/CodeGen/AArch64/stack-probing-64k.ll | 2 +- .../CodeGen/AArch64/stack-probing-dynamic.ll | 16 +- .../test/CodeGen/AArch64/stack-probing-sve.ll | 10 +- llvm/test/CodeGen/AArch64/stack-probing.ll | 2 +- llvm/test/CodeGen/AArch64/vselect-ext.ll | 178 +- .../GlobalISel/irtranslator-prefetch.ll | 32 + .../AMDGPU/codegen-internal-only-func.ll | 10 +- llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll | 12 +- .../CodeGen/AMDGPU/integer-mad-patterns.ll | 1560 ++++++++- ...ne-sink-temporal-divergence-swdev407790.ll | 39 +- .../CodeGen/AMDGPU/regpressure_printer.mir | 171 +- .../AMDGPU/tid-mul-func-xnack-all-any.ll | 17 +- .../tid-mul-func-xnack-all-not-supported.ll | 17 +- .../AMDGPU/tid-mul-func-xnack-all-off.ll | 17 +- .../AMDGPU/tid-mul-func-xnack-all-on.ll | 17 +- .../AMDGPU/tid-mul-func-xnack-any-off-1.ll | 17 +- .../AMDGPU/tid-mul-func-xnack-any-off-2.ll | 17 +- .../AMDGPU/tid-mul-func-xnack-any-on-1.ll | 17 +- .../AMDGPU/tid-mul-func-xnack-any-on-2.ll | 17 +- .../CodeGen/AMDGPU/tid-one-func-xnack-any.ll | 17 +- .../tid-one-func-xnack-not-supported.ll | 17 +- .../CodeGen/AMDGPU/tid-one-func-xnack-off.ll | 17 +- .../CodeGen/AMDGPU/tid-one-func-xnack-on.ll | 17 +- llvm/test/CodeGen/ARM/aapcs-hfa-code.ll | 12 +- llvm/test/CodeGen/ARM/ha-alignstack-call.ll | 20 +- llvm/test/CodeGen/ARM/vector-store.ll | 17 +- .../BPF/GlobalISel/ir-translator-ret.ll | 7 + llvm/test/CodeGen/DirectX/UAVMetadata.ll | 20 +- llvm/test/CodeGen/DirectX/cbuf.ll | 2 +- .../CodeGen/DirectX/legacy_cb_layout_0.ll | 2 +- .../CodeGen/DirectX/legacy_cb_layout_1.ll | 2 +- .../CodeGen/DirectX/legacy_cb_layout_2.ll | 4 +- .../CodeGen/DirectX/legacy_cb_layout_3.ll | 2 +- .../Generic/opt-codegen-no-target-machine.ll | 2 +- llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll | 804 +++++ llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll | 804 +++++ llvm/test/CodeGen/LoongArch/lasx/fsqrt.ll | 65 + .../LoongArch/lasx/ir-instruction/fcmp.ll | 692 ++++ .../LoongArch/lasx/ir-instruction/fdiv.ll | 29 + .../LoongArch/lasx/ir-instruction/fneg.ll | 29 + .../LoongArch/lasx/ir-instruction/icmp.ll | 939 +++++ llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll | 804 +++++ llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll | 804 +++++ llvm/test/CodeGen/LoongArch/lsx/fsqrt.ll | 65 + .../LoongArch/lsx/ir-instruction/fcmp.ll | 692 ++++ .../LoongArch/lsx/ir-instruction/fdiv.ll | 29 + .../LoongArch/lsx/ir-instruction/fneg.ll | 29 + .../LoongArch/lsx/ir-instruction/icmp.ll | 939 +++++ .../beqc-bnec-register-constraint.ll | 4 +- llvm/test/CodeGen/Mips/pr49200.ll | 9 +- llvm/test/CodeGen/NVPTX/bf16-instructions.ll | 103 + .../test/CodeGen/NVPTX/bf16x2-instructions.ll | 10 + llvm/test/CodeGen/PowerPC/aix-ehinfo-sym.ll | 13 +- ...ix-emit-tracebacktable-clobber-register.ll | 2 +- llvm/test/CodeGen/PowerPC/aix-exception.ll | 2 +- .../test/CodeGen/PowerPC/aix-tls-gd-double.ll | 32 +- .../PowerPC/aix-tls-gd-target-flags.ll | 4 +- llvm/test/CodeGen/PowerPC/ctrloops-pseudo.ll | 4 +- .../PowerPC/pcrel-tls-local-dynamic.ll | 4 +- .../CodeGen/PowerPC/pcrel-tls-local-exec.ll | 20 +- .../PowerPC/pcrel-tls_get_addr_clobbers.ll | 2 +- llvm/test/CodeGen/PowerPC/tls-crash.mir | 4 +- .../RISCV/GlobalISel/irtranslator/vacopy.ll | 28 + .../RISCV/GlobalISel/irtranslator/vararg.ll | 1412 +++++++- .../legalizer/legalize-vaarg-rv32.mir | 70 + .../legalizer/legalize-vaarg-rv64.mir | 93 + .../GlobalISel/legalizer/legalize-vacopy.mir | 23 + .../legalizer/legalize-vastart-rv32.mir | 29 + .../legalizer/legalize-vastart-rv64.mir | 29 + llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll | 1008 ++++++ llvm/test/CodeGen/RISCV/llvm.frexp.ll | 285 +- .../CodeGen/RISCV/macro-fusions-veyron-v1.mir | 159 + .../CodeGen/RISCV/relax-per-target-feature.ll | 34 + .../CodeGen/RISCV/riscv-codegenprepare-asm.ll | 3 +- .../RISCV/rvv/fixed-vectors-int-shuffles.ll | 10 +- llvm/test/CodeGen/RISCV/sextw-removal.ll | 67 +- .../SPIRV/opencl/basic/get_global_offset.ll | 8 +- llvm/test/CodeGen/SPIRV/struct.ll | 3 +- llvm/test/CodeGen/SystemZ/atomicrmw-add-04.ll | 61 +- llvm/test/CodeGen/SystemZ/atomicrmw-and-03.ll | 2 +- llvm/test/CodeGen/SystemZ/atomicrmw-and-04.ll | 9 +- .../CodeGen/SystemZ/atomicrmw-minmax-03.ll | 304 +- .../CodeGen/SystemZ/atomicrmw-minmax-04.ll | 245 +- .../test/CodeGen/SystemZ/atomicrmw-nand-03.ll | 22 +- .../test/CodeGen/SystemZ/atomicrmw-nand-04.ll | 87 +- llvm/test/CodeGen/SystemZ/atomicrmw-or-04.ll | 11 +- llvm/test/CodeGen/SystemZ/atomicrmw-sub-04.ll | 61 +- .../test/CodeGen/SystemZ/atomicrmw-xchg-03.ll | 2 +- .../test/CodeGen/SystemZ/atomicrmw-xchg-04.ll | 2 +- llvm/test/CodeGen/SystemZ/atomicrmw-xor-04.ll | 9 +- llvm/test/CodeGen/SystemZ/xor-04.ll | 4 +- .../test/CodeGen/Thumb2/mve-blockplacement.ll | 12 +- .../CodeGen/WebAssembly/wasm-eh-prepare.ll | 4 +- llvm/test/CodeGen/WinEH/wineh-asm.ll | 2 +- llvm/test/CodeGen/WinEH/wineh-cloning.ll | 2 +- llvm/test/CodeGen/WinEH/wineh-demotion.ll | 2 +- llvm/test/CodeGen/WinEH/wineh-no-demotion.ll | 2 +- .../CodeGen/X86/2011-10-19-widen_vselect.ll | 5 +- llvm/test/CodeGen/X86/2012-07-10-extload64.ll | 2 +- llvm/test/CodeGen/X86/atomic-nocx16.ll | 49 + .../CodeGen/X86/atomic-non-integer-fp128.ll | 152 +- llvm/test/CodeGen/X86/atomic-non-integer.ll | 491 ++- llvm/test/CodeGen/X86/atomic128.ll | 576 +--- llvm/test/CodeGen/X86/atomicf128.ll | 46 - llvm/test/CodeGen/X86/avx512fp16-arith.ll | 8 +- llvm/test/CodeGen/X86/code-model-elf.ll | 238 +- .../test/CodeGen/X86/combine-concatvectors.ll | 3 +- llvm/test/CodeGen/X86/dwarf-eh-prepare-dbg.ll | 4 +- llvm/test/CodeGen/X86/dwarf-eh-prepare.ll | 4 +- llvm/test/CodeGen/X86/dwarf_eh_resume.ll | 4 +- .../test/CodeGen/X86/evex-to-vex-compress.mir | 1580 ++++----- llvm/test/CodeGen/X86/fold-load-vec.ll | 2 +- llvm/test/CodeGen/X86/fold-pcmpeqd-2.ll | 18 +- llvm/test/CodeGen/X86/fp-intrinsics.ll | 4 +- llvm/test/CodeGen/X86/gep-expanded-vector.ll | 46 +- llvm/test/CodeGen/X86/gfni-funnel-shifts.ll | 8 +- llvm/test/CodeGen/X86/gfni-rotates.ll | 8 +- llvm/test/CodeGen/X86/half.ll | 14 +- llvm/test/CodeGen/X86/ldexp.ll | 7 +- llvm/test/CodeGen/X86/llvm.frexp.ll | 467 ++- llvm/test/CodeGen/X86/memset64-on-x86-32.ll | 31 +- .../CodeGen/X86/min-legal-vector-width.ll | 4 +- llvm/test/CodeGen/X86/nocx16.ll | 21 - llvm/test/CodeGen/X86/nontemporal-3.ll | 646 +--- llvm/test/CodeGen/X86/pr38738.ll | 25 +- llvm/test/CodeGen/X86/pr41619.ll | 3 +- llvm/test/CodeGen/X86/pr74736.ll | 68 + llvm/test/CodeGen/X86/slow-unaligned-mem.ll | 94 +- llvm/test/CodeGen/X86/var-permute-256.ll | 166 +- llvm/test/CodeGen/X86/vec_fcopysign.ll | 59 +- llvm/test/CodeGen/X86/vec_zero_cse.ll | 2 +- llvm/test/CodeGen/X86/vector-fshl-128.ll | 12 +- llvm/test/CodeGen/X86/vector-fshl-256.ll | 12 +- llvm/test/CodeGen/X86/vector-fshl-512.ll | 12 +- llvm/test/CodeGen/X86/vector-fshl-rot-128.ll | 12 +- llvm/test/CodeGen/X86/vector-fshl-rot-256.ll | 20 +- llvm/test/CodeGen/X86/vector-fshl-rot-512.ll | 36 +- llvm/test/CodeGen/X86/vector-fshr-128.ll | 12 +- llvm/test/CodeGen/X86/vector-fshr-256.ll | 12 +- llvm/test/CodeGen/X86/vector-fshr-512.ll | 12 +- llvm/test/CodeGen/X86/vector-fshr-rot-128.ll | 12 +- llvm/test/CodeGen/X86/vector-fshr-rot-256.ll | 24 +- llvm/test/CodeGen/X86/vector-fshr-rot-512.ll | 48 +- .../vector-interleaved-load-i8-stride-4.ll | 1418 ++++---- .../vector-interleaved-load-i8-stride-6.ll | 3029 ++++++++--------- .../vector-interleaved-store-i16-stride-3.ll | 9 +- .../vector-interleaved-store-i16-stride-5.ll | 310 +- .../vector-interleaved-store-i16-stride-6.ll | 144 +- .../vector-interleaved-store-i16-stride-7.ll | 89 +- .../vector-interleaved-store-i8-stride-3.ll | 8 +- .../vector-interleaved-store-i8-stride-5.ll | 504 ++- .../vector-interleaved-store-i8-stride-6.ll | 138 +- .../vector-interleaved-store-i8-stride-7.ll | 474 +-- llvm/test/CodeGen/X86/vector-rotate-128.ll | 8 +- llvm/test/CodeGen/X86/vector-rotate-256.ll | 16 +- llvm/test/CodeGen/X86/vector-rotate-512.ll | 48 +- .../test/CodeGen/X86/vector-shift-ashr-128.ll | 2 +- .../test/CodeGen/X86/vector-shift-ashr-256.ll | 2 +- .../test/CodeGen/X86/vector-shift-lshr-128.ll | 2 +- .../test/CodeGen/X86/vector-shift-lshr-256.ll | 2 +- llvm/test/CodeGen/X86/vector-shift-shl-128.ll | 2 +- llvm/test/CodeGen/X86/vector-shift-shl-256.ll | 2 +- .../CodeGen/X86/vector-shuffle-combining.ll | 2 +- llvm/test/CodeGen/X86/vector-shuffle-v192.ll | 52 +- .../CodeGen/X86/x86-interleaved-access.ll | 8 +- llvm/test/CodeGen/X86/zero-remat.ll | 7 +- .../assignment-tracking/X86/large-type.ll | 58 + .../OrcV2Examples/Inputs/argc_sub1_elf.ll | 1 - .../global_metadata_code_model.ll | 10 + .../AddressSanitizer/global_with_comdat.ll | 4 +- .../InstrProfiling/icall-comdat.ll | 34 +- .../InstrProfiling/platform.ll | 2 +- llvm/test/MC/AArch64/elf-reloc-ptrauth.s | 16 +- llvm/test/MC/AMDGPU/exp.s | 21 + llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s | 187 +- llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err.s | 27 +- llvm/test/MC/AMDGPU/gfx12_asm_exp.s | 61 + llvm/test/MC/AMDGPU/gfx12_asm_sop1.s | 3 + llvm/test/MC/AMDGPU/sopp-err.s | 107 +- .../MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt | 13 + .../Disassembler/AMDGPU/gfx12_dasm_sop1.txt | 3 + .../AMDGPU/gfx12_dasm_vinterp.txt | 251 ++ llvm/test/MC/Disassembler/X86/apx/invpcid.txt | 6 + llvm/test/MC/ELF/reloc-directive.s | 4 + llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s | 298 ++ llvm/test/MC/PowerPC/ppc64-errors.s | 4 +- llvm/test/MC/X86/apx/invpcid-att.s | 8 + llvm/test/MC/X86/apx/invpcid-intel.s | 5 + llvm/test/MachineVerifier/test_g_prefetch.mir | 40 + llvm/test/TableGen/ContextlessPredicates.td | 2 +- llvm/test/TableGen/DefaultOpsGlobalISel.td | 18 +- .../builtins/match-table-replacerreg.td | 20 +- .../match-table-imms.td | 28 +- .../match-table-patfrag-root.td | 2 +- .../GlobalISelCombinerEmitter/match-table.td | 62 +- .../GlobalISelEmitter-input-discard.td | 2 +- ...obalISelEmitter-multiple-output-discard.td | 8 +- .../GlobalISelEmitter-multiple-output.td | 12 +- .../GlobalISelEmitter-nested-subregs.td | 4 +- .../GlobalISelEmitter-output-discard.td | 2 +- .../TableGen/GlobalISelEmitter-zero-reg.td | 4 +- llvm/test/TableGen/GlobalISelEmitter.td | 131 +- .../GlobalISelEmitterCustomPredicate.td | 10 +- .../test/TableGen/GlobalISelEmitterHwModes.td | 8 +- .../GlobalISelEmitterMatchTableOptimizer.td | 2 +- ...rMatchTableOptimizerSameOperand-invalid.td | 4 +- .../TableGen/GlobalISelEmitterRegSequence.td | 4 +- llvm/test/TableGen/GlobalISelEmitterSubreg.td | 16 +- llvm/test/TableGen/gisel-physreg-input.td | 8 +- .../alignment-from-assumptions-track-users.ll | 204 ++ .../IRCE/non-loop-invariant-rhs-instr.ll | 2 +- .../promote-iv-to-eliminate-casts.ll | 2 +- llvm/test/Transforms/InstCombine/alloca.ll | 11 + .../InstCombine/binop-of-displaced-shifts.ll | 29 + .../InstCombine/ctlz-cttz-shifts.ll | 234 ++ llvm/test/Transforms/InstCombine/icmp-gep.ll | 46 + ...al-to-icmp-eq-of-lshr-val-by-bits-and-0.ll | 2 +- ...al-to-icmp-ne-of-lshr-val-by-bits-and-0.ll | 2 +- .../test/Transforms/InstCombine/known-bits.ll | 242 ++ llvm/test/Transforms/InstCombine/not.ll | 23 +- llvm/test/Transforms/InstCombine/or.ll | 13 + llvm/test/Transforms/InstSimplify/returned.ll | 25 + .../AArch64/binopshuffles-inseltpoison.ll | 1 + .../AArch64/binopshuffles.ll | 1 + .../AArch64/fixed-deinterleave-intrinsics.ll | 2 + ...aved-accesses-extract-user-inseltpoison.ll | 1 + .../interleaved-accesses-extract-user.ll | 1 + .../interleaved-accesses-inseltpoison.ll | 2 + .../AArch64/interleaved-accesses.ll | 2 + .../scalable-deinterleave-intrinsics.ll | 1 + .../AArch64/sve-interleaved-accesses.ll | 1 + ...aved-accesses-extract-user-inseltpoison.ll | 1 + .../ARM/interleaved-accesses-extract-user.ll | 1 + .../ARM/interleaved-accesses-inseltpoison.ll | 3 + .../ARM/interleaved-accesses.ll | 3 + .../RISCV/interleaved-accesses.ll | 2 + .../InterleavedAccess/RISCV/zve32x.ll | 2 + .../InterleavedAccess/RISCV/zvl32b.ll | 2 + ...interleave-load-extract-shuffle-changes.ll | 1 + ...leaved-accesses-64bits-avx-inseltpoison.ll | 1 + .../X86/interleaved-accesses-64bits-avx.ll | 1 + .../X86/interleavedLoad-inseltpoison.ll | 2 + .../InterleavedAccess/X86/interleavedLoad.ll | 2 + .../X86/interleavedStore-inseltpoison.ll | 1 + .../InterleavedAccess/X86/interleavedStore.ll | 1 + llvm/test/Transforms/LICM/pr64897.ll | 8 +- llvm/test/Transforms/LoopPredication/basic.ll | 2 +- .../LoopStrengthReduce/lsr-term-fold.ll | 116 +- .../LoopUnroll/loop-branch-folding.ll | 936 +++++ .../runtime-exit-phi-scev-invalidation.ll | 2 +- .../AArch64/eliminate-tail-predication.ll | 14 +- .../gather-do-not-vectorize-addressing.ll | 4 +- .../LoopVectorize/AArch64/masked-call.ll | 56 +- .../AArch64/outer_loop_prefer_scalable.ll | 4 +- .../AArch64/pr60831-sve-inv-store-crash.ll | 34 +- .../AArch64/scalable-avoid-scalarization.ll | 38 +- .../AArch64/scalable-reduction-inloop-cond.ll | 66 +- .../AArch64/scalable-strict-fadd.ll | 88 +- .../AArch64/sve-cond-inv-loads.ll | 66 +- .../sve-epilog-vect-inloop-reductions.ll | 4 +- .../AArch64/sve-epilog-vect-reductions.ll | 4 +- .../sve-epilog-vect-strict-reductions.ll | 46 +- .../LoopVectorize/AArch64/sve-epilog-vect.ll | 228 +- .../LoopVectorize/AArch64/sve-fneg.ll | 40 +- .../AArch64/sve-gather-scatter.ll | 108 +- .../AArch64/sve-inductions-unusual-types.ll | 12 +- .../LoopVectorize/AArch64/sve-inductions.ll | 36 +- .../AArch64/sve-interleaved-accesses.ll | 70 +- .../sve-interleaved-masked-accesses.ll | 24 +- .../LoopVectorize/AArch64/sve-inv-store.ll | 44 +- .../AArch64/sve-live-out-pointer-induction.ll | 4 +- .../AArch64/sve-low-trip-count.ll | 8 +- .../LoopVectorize/AArch64/sve-multiexit.ll | 100 +- .../sve-runtime-check-size-based-threshold.ll | 4 +- .../AArch64/sve-tail-folding-forced.ll | 7 +- .../AArch64/sve-tail-folding-optsize.ll | 26 +- .../sve-tail-folding-overflow-checks.ll | 8 +- .../AArch64/sve-tail-folding-reductions.ll | 24 +- .../AArch64/sve-tail-folding-unroll.ll | 8 +- .../LoopVectorize/AArch64/sve-tail-folding.ll | 40 +- .../AArch64/sve-vector-reverse.ll | 8 +- .../LoopVectorize/AArch64/sve-vfabi.ll | 8 +- .../LoopVectorize/AArch64/sve-widen-gep.ll | 11 +- .../LoopVectorize/AArch64/sve-widen-phi.ll | 16 +- .../AArch64/synthesize-mask-for-call.ll | 18 +- .../AArch64/tail-folding-styles.ll | 20 +- .../AArch64/type-shrinkage-zext-costs.ll | 36 +- .../AArch64/uniform-args-call-variants.ll | 8 +- .../AArch64/vector-call-linear-args.ll | 44 +- .../widen-call-with-intrinsic-or-libfunc.ll | 7 +- .../AArch64/wider-VF-for-callinst.ll | 4 +- .../LoopVectorize/Hexagon/maximum-vf-crash.ll | 3 +- .../LoopVectorize/Hexagon/minimum-vf.ll | 3 +- .../LoopVectorize/RISCV/defaults.ll | 38 +- .../Transforms/LoopVectorize/RISCV/divrem.ll | 198 +- .../LoopVectorize/RISCV/inloop-reduction.ll | 48 +- .../RISCV/interleaved-accesses.ll | 8 +- .../Transforms/LoopVectorize/RISCV/lmul.ll | 14 +- .../LoopVectorize/RISCV/low-trip-count.ll | 28 +- .../LoopVectorize/RISCV/mask-index-type.ll | 44 +- .../RISCV/masked_gather_scatter.ll | 8 +- .../RISCV/riscv-vector-reverse.ll | 14 +- .../LoopVectorize/RISCV/safe-dep-distance.ll | 66 +- .../LoopVectorize/RISCV/scalable-basics.ll | 48 +- .../LoopVectorize/RISCV/scalable-tailfold.ll | 120 +- .../RISCV/select-cmp-reduction.ll | 24 +- .../LoopVectorize/RISCV/short-trip-count.ll | 8 +- .../LoopVectorize/RISCV/strided-accesses.ll | 32 +- .../LoopVectorize/RISCV/uniform-load-store.ll | 60 +- .../first-order-recurrence-chains-vplan.ll | 6 +- ...-order-recurrence-sink-replicate-region.ll | 18 +- .../Transforms/LoopVectorize/icmp-uniforms.ll | 3 +- .../interleave-and-scalarize-only.ll | 11 +- .../LoopVectorize/outer_loop_scalable.ll | 4 +- .../LoopVectorize/reduction-inloop-cond.ll | 837 ++++- .../LoopVectorize/reduction-small-size.ll | 5 +- .../scalable-first-order-recurrence.ll | 1 + .../LoopVectorize/scalable-inductions.ll | 130 +- .../LoopVectorize/scalable-lifetime.ll | 28 +- ...able-loop-unpredicated-body-scalar-tail.ll | 8 +- .../scalable-reduction-inloop.ll | 6 +- .../scalable-trunc-min-bitwidth.ll | 8 +- .../LoopVectorize/vplan-dot-printing.ll | 4 +- .../LoopVectorize/vplan-iv-transforms.ll | 3 +- .../LoopVectorize/vplan-printing.ll | 75 +- .../vplan-sink-scalars-and-merge-vf1.ll | 3 +- .../vplan-sink-scalars-and-merge.ll | 42 +- llvm/test/Transforms/MemCpyOpt/pr75010.ll | 19 + .../test/Transforms/PhaseOrdering/X86/vdiv.ll | 47 +- llvm/test/Transforms/Reassociate/local-cse.ll | 40 +- .../Transforms/Reassociate/reassoc-mul-nuw.ll | 34 + .../RISCV/remarks-insert-into-small-vector.ll | 56 + .../latency/memory-annotations-unsupported.s | 2 +- .../snippet-address-annotations-unsupported.s | 9 + .../latency/subprocess-address-annotation.s | 26 + llvm/test/tools/llvm-objcopy/regex-error.test | 13 + .../ELF/dynamic-section-machine-specific.test | 9 + .../ELF/AArch64/aarch64-feature-pauth.s | 98 + .../ELF/broken-dynamic-reloc.test | 54 +- .../ELF/dynamic-tags-machine-specific.test | 32 +- .../ELF/machine-specific-section-types.test | 5 + .../tools/llvm-readobj/ELF/needed-libs.test | 13 + .../tools/llvm-readobj/ELF/relr-relocs.test | 17 +- llvm/test/tools/obj2yaml/XCOFF/aix.yaml | 38 + .../tools/obj2yaml/XCOFF/aux-symbols.yaml | 322 ++ llvm/tools/bugpoint-passes/CMakeLists.txt | 2 +- llvm/tools/dsymutil/DebugMap.cpp | 2 +- llvm/tools/llvm-c-test/echo.cpp | 46 +- llvm/tools/llvm-cov/CodeCoverage.cpp | 2 +- llvm/tools/llvm-cov/CoverageReport.cpp | 2 +- .../tools/llvm-cov/SourceCoverageViewHTML.cpp | 2 +- .../tools/llvm-exegesis/lib/BenchmarkResult.h | 3 + .../llvm-exegesis/lib/BenchmarkRunner.cpp | 35 +- .../tools/llvm-exegesis/lib/BenchmarkRunner.h | 2 +- .../lib/LatencyBenchmarkRunner.cpp | 5 +- .../lib/LatencyBenchmarkRunner.h | 4 +- llvm/tools/llvm-exegesis/lib/SnippetFile.cpp | 13 + llvm/tools/llvm-exegesis/lib/Target.cpp | 10 +- llvm/tools/llvm-exegesis/lib/Target.h | 4 +- llvm/tools/llvm-exegesis/llvm-exegesis.cpp | 28 +- llvm/tools/llvm-ml/llvm-ml.cpp | 2 +- llvm/tools/llvm-readobj/ELFDumper.cpp | 121 +- llvm/tools/llvm-readtapi/llvm-readtapi.cpp | 37 +- llvm/tools/llvm-shlib/CMakeLists.txt | 18 +- llvm/tools/obj2yaml/xcoff2yaml.cpp | 254 ++ llvm/tools/opt/opt.cpp | 3 +- llvm/unittests/Analysis/ValueLatticeTest.cpp | 1 - .../Analysis/VectorFunctionABITest.cpp | 573 ++-- llvm/unittests/Analysis/VectorUtilsTest.cpp | 1 - .../DebugInfo/DWARF/DWARFDebugInfoTest.cpp | 36 + .../ExecutionEngine/Orc/CMakeLists.txt | 1 + .../ExecutionEngine/Orc/OrcCAPITest.cpp | 85 + llvm/unittests/Support/Chrono.cpp | 1 - llvm/unittests/Support/CommandLineTest.cpp | 74 +- .../unittests/Support/InstructionCostTest.cpp | 1 - llvm/unittests/Support/MemoryTest.cpp | 1 - llvm/unittests/Support/Path.cpp | 12 +- llvm/unittests/TableGen/AutomataTest.cpp | 1 - .../TargetParser/TargetParserTest.cpp | 18 +- llvm/unittests/TextAPI/RecordTests.cpp | 6 +- .../llvm-exegesis/X86/SnippetFileTest.cpp | 10 + llvm/utils/TableGen/GlobalISelEmitter.cpp | 56 +- llvm/utils/TableGen/GlobalISelMatchTable.cpp | 2 +- .../GlobalISelMatchTableExecutorEmitter.h | 1 - llvm/utils/TableGen/IntrinsicEmitter.cpp | 1 - llvm/utils/TableGen/X86DisassemblerTables.cpp | 271 +- llvm/utils/git/code-format-helper.py | 221 +- .../clang-tidy/hicpp/BUILD.gn | 1 + .../ExecutionEngine/Orc/Debugging/BUILD.gn | 3 +- .../gn/secondary/llvm/lib/Target/BPF/BUILD.gn | 4 + .../unittests/ExecutionEngine/Orc/BUILD.gn | 1 + mlir/cmake/modules/MLIRConfig.cmake.in | 1 + mlir/include/mlir-c/IR.h | 3 + .../mlir/Dialect/Affine/IR/AffineOps.td | 14 +- .../include/mlir/Dialect/Arith/IR/ArithOps.td | 32 +- .../mlir/Dialect/ArmSME/Transforms/Passes.td | 5 +- .../mlir/Dialect/Complex/IR/ComplexOps.td | 2 +- mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td | 23 + .../mlir/Dialect/LLVMIR/LLVMInterfaces.td | 57 + mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td | 34 +- .../Linalg/TransformOps/LinalgTransformOps.td | 9 +- .../Dialect/Linalg/Transforms/Transforms.h | 2 + mlir/include/mlir/Dialect/Math/IR/MathOps.td | 29 + mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td | 29 +- .../mlir/Dialect/OpenACC/OpenACCOps.td | 2 +- .../mlir/Dialect/SCF/Transforms/Transforms.h | 7 + .../mlir/Dialect/SPIRV/IR/SPIRVBitOps.td | 8 + .../Vector/Transforms/VectorTransforms.h | 10 - mlir/include/mlir/IR/BuiltinAttributes.td | 12 +- .../mlir/IR/BuiltinLocationAttributes.td | 13 +- mlir/include/mlir/IR/BuiltinTypes.td | 43 +- mlir/include/mlir/IR/OpImplementation.h | 13 + mlir/include/mlir/Pass/Pass.h | 47 + .../include/mlir/Target/LLVMIR/ModuleImport.h | 6 + .../DataFlow/IntegerRangeAnalysis.cpp | 2 +- .../Bindings/Python/ExecutionEngineModule.cpp | 2 +- mlir/lib/Bindings/Python/IRInterfaces.cpp | 2 +- mlir/lib/Bindings/Python/IRModule.h | 6 +- mlir/lib/Bindings/Python/Pass.cpp | 3 +- mlir/lib/CAPI/IR/IR.cpp | 4 + .../Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp | 6 - .../GPUCommon/GPUToLLVMConversion.cpp | 12 +- mlir/lib/Conversion/MathToLibm/MathToLibm.cpp | 1 + .../MapMemRefStorageClassPass.cpp | 2 +- .../PDLToPDLInterp/PredicateTree.cpp | 2 +- .../Conversion/VectorToGPU/VectorToGPU.cpp | 15 +- mlir/lib/Dialect/Affine/IR/AffineOps.cpp | 11 + mlir/lib/Dialect/Affine/IR/CMakeLists.txt | 1 + mlir/lib/Dialect/Affine/Utils/CMakeLists.txt | 1 + mlir/lib/Dialect/Complex/IR/ComplexOps.cpp | 10 +- mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp | 8 +- .../LLVMIR/Transforms/TypeConsistency.cpp | 5 +- .../TransformOps/LinalgTransformOps.cpp | 36 + .../Linalg/Transforms/ElementwiseOpFusion.cpp | 44 +- mlir/lib/Dialect/Math/IR/MathOps.cpp | 18 + mlir/lib/Dialect/Mesh/IR/MeshOps.cpp | 13 +- mlir/lib/Dialect/SCF/IR/CMakeLists.txt | 3 +- mlir/lib/Dialect/SCF/IR/SCF.cpp | 132 +- .../Dialect/SCF/Transforms/LoopPipelining.cpp | 137 +- .../SPIRV/IR/SPIRVCanonicalization.cpp | 189 + mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp | 49 - .../SparseTensor/IR/SparseTensorDialect.cpp | 20 +- .../SparseTensor/Transforms/LoopEmitter.cpp | 188 +- .../SparseTensor/Transforms/LoopEmitter.h | 39 +- .../Transforms/SparseBufferRewriting.cpp | 2 +- .../Transforms/SparseGPUCodegen.cpp | 4 + .../Transforms/SparseReinterpretMap.cpp | 20 +- .../Transforms/Sparsification.cpp | 24 +- mlir/lib/Dialect/Tensor/IR/TensorOps.cpp | 134 +- mlir/lib/Dialect/Utils/IndexingUtils.cpp | 2 +- mlir/lib/Dialect/Vector/IR/VectorOps.cpp | 10 + .../Transforms/LowerVectorTranspose.cpp | 34 +- mlir/lib/IR/AsmPrinter.cpp | 88 +- mlir/lib/Pass/Pass.cpp | 10 + mlir/lib/Pass/PassDetail.h | 20 - mlir/lib/Target/LLVMIR/ModuleImport.cpp | 13 + .../Utils/GreedyPatternRewriteDriver.cpp | 3 - mlir/python/CMakeLists.txt | 3 +- mlir/python/mlir/dialects/_ods_common.py | 4 + mlir/python/mlir/dialects/affine.py | 105 +- mlir/test/CAPI/ir.c | 9 + .../MathToLibm/convert-to-libm.mlir | 39 + .../VectorToGPU/vector-to-mma-ops.mlir | 30 + .../Dialect/Arith/int-range-interface.mlir | 26 + mlir/test/Dialect/Complex/ops.mlir | 3 + mlir/test/Dialect/LLVMIR/roundtrip.mlir | 10 + .../test/Dialect/LLVMIR/type-consistency.mlir | 14 + .../Dialect/Linalg/transform-op-match.mlir | 38 + .../test/Dialect/Linalg/transpose-conv2d.mlir | 2 +- .../MemRef/expand-strided-metadata.mlir | 20 + mlir/test/Dialect/Mesh/canonicalization.mlir | 2 +- mlir/test/Dialect/Mesh/invalid.mlir | 56 +- mlir/test/Dialect/Mesh/ops.mlir | 15 +- .../Dialect/Mesh/sharding-propagation.mlir | 2 +- .../NVGPU/transform-pipeline-shared.mlir | 26 +- mlir/test/Dialect/OpenACC/ops.mlir | 4 +- mlir/test/Dialect/SCF/canonicalize.mlir | 50 - mlir/test/Dialect/SCF/loop-pipelining.mlir | 47 + mlir/test/Dialect/SPIRV/IR/bit-ops.mlir | 6 +- .../SPIRV/Transforms/canonicalize.mlir | 366 ++ .../sparse_conv_2d_slice_based.mlir | 411 +-- .../Dialect/SparseTensor/spy_sddmm_bsr.mlir | 103 + mlir/test/Dialect/Tensor/canonicalize.mlir | 17 + .../Dialect/Tensor/invalid-canonicalize.mlir | 15 - mlir/test/Dialect/Tensor/invalid.mlir | 13 - mlir/test/Dialect/Vector/invalid.mlir | 22 + mlir/test/IR/custom-print-parse.mlir | 17 + mlir/test/IR/invalid-custom-print-parse.mlir | 16 + .../SparseTensor/CPU/dual_sparse_conv_2d.mlir | 12 +- .../CPU/sparse_conv_1d_nwc_wcf.mlir | 1 - .../SparseTensor/CPU/sparse_conv_2d.mlir | 33 +- .../SparseTensor/CPU/sparse_conv_2d_55.mlir | 220 ++ .../CPU/sparse_conv_2d_nchw_fchw.mlir | 1 - .../SparseTensor/CPU/sparse_conv_3d.mlir | 62 +- .../CPU/sparse_conv_3d_ndhwc_dhwcf.mlir | 1 - mlir/test/Target/LLVMIR/Import/nsw_nuw.ll | 14 + mlir/test/Target/LLVMIR/nsw_nuw.mlir | 14 + mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp | 1 + mlir/test/lib/Dialect/Test/TestOps.td | 14 + .../lib/Transforms/TestIntRangeInference.cpp | 2 + mlir/test/python/dialects/affine.py | 199 +- mlir/unittests/Pass/CMakeLists.txt | 1 + mlir/unittests/Pass/PassManagerTest.cpp | 100 + openmp/libomptarget/CMakeLists.txt | 21 +- openmp/libomptarget/DeviceRTL/CMakeLists.txt | 2 +- openmp/libomptarget/include/PluginManager.h | 21 + .../plugins-nextgen/common/CMakeLists.txt | 3 + .../common/src/GlobalHandler.cpp | 25 +- .../common/src/PluginInterface.cpp | 21 +- openmp/libomptarget/src/CMakeLists.txt | 3 + openmp/libomptarget/src/PluginManager.cpp | 2 +- openmp/libomptarget/src/interface.cpp | 3 + openmp/libomptarget/src/rtl.cpp | 1 + openmp/libomptarget/test/Inputs/empty.c | 1 - .../test/offloading/back2back_distribute.c | 12 +- .../test/offloading/barrier_fence.c | 2 - .../libomptarget/test/offloading/bug60119.c | 6 - .../libomptarget/test/offloading/bug74582.c | 13 + .../test/offloading/shared_lib_fp_mapping.c | 4 +- .../offloading/target_map_for_member_data.cpp | 25 + openmp/runtime/src/z_Linux_util.cpp | 2 +- polly/lib/Analysis/ScopBuilder.cpp | 78 +- .../llvm-project-overlay/llvm/BUILD.bazel | 36 +- .../llvm/unittests/BUILD.bazel | 1 + .../llvm-project-overlay/mlir/BUILD.bazel | 1 - .../mlir/unittests/BUILD.bazel | 1 + 1438 files changed, 45689 insertions(+), 17273 deletions(-) create mode 100644 .github/workflows/spirv-tests.yml create mode 100644 bolt/test/X86/Inputs/blarge_profile_stale.std-hash.yaml create mode 100644 bolt/test/X86/reader-stale-yaml-std.test create mode 100644 clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.cpp create mode 100644 clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.h create mode 100644 clang-tools-extra/docs/clang-tidy/checks/hicpp/ignored-remove-result.rst create mode 100644 clang-tools-extra/test/clang-tidy/checkers/hicpp/ignored-remove-result.cpp create mode 100644 clang/include/clang/Basic/TargetOSMacros.def create mode 100644 clang/test/AST/ms-constexpr.cpp create mode 100644 clang/test/C/C2x/n2836_n2939.c delete mode 100644 clang/test/CXX/drs/dr2354.cpp delete mode 100644 clang/test/CXX/drs/dr2406.cpp create mode 100644 clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c create mode 100644 clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_qrshr.c create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create2_bool.c create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create4_bool.c create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get2_bool.c create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get4_bool.c create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set2_bool.c create mode 100644 clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set4_bool.c create mode 100644 clang/test/CodeGen/cx-complex-range.c create mode 100644 clang/test/CodeGen/pragma-cx-limited-range.c create mode 100644 clang/test/CodeGen/scoped-atomic-ops.c create mode 100644 clang/test/CodeGen/tbaa.c create mode 100644 clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl create mode 100644 clang/test/Driver/fdefine-target-os-macros.c create mode 100644 clang/test/Driver/range.c create mode 100644 clang/test/ParserOpenACC/parse-wait-construct.c create mode 100644 clang/test/Sema/scoped-atomic-ops.c create mode 100644 clang/test/Sema/switch-default.c create mode 100644 clang/test/SemaCXX/ms-constexpr-invalid.cpp create mode 100644 clang/test/SemaCXX/ms-constexpr-new.cpp create mode 100644 clang/test/SemaCXX/ms-constexpr.cpp create mode 100644 clang/test/SemaCXX/warn-unsafe-buffer-usage-fixits-add-assign.cpp create mode 100644 clang/test/SemaTemplate/GH71595.cpp create mode 100644 compiler-rt/test/hwasan/TestCases/Linux/fixed-shadow.c create mode 100644 compiler-rt/test/orc/TestCases/Darwin/arm64/Inputs/ret_self.S create mode 100644 compiler-rt/test/orc/TestCases/Darwin/arm64/trivial-dlsym.c create mode 100644 compiler-rt/test/orc/TestCases/Darwin/x86-64/Inputs/ret_self.S create mode 100644 compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-dlsym.c create mode 100644 compiler-rt/test/profile/ContinuousSyncMode/reset-default-profile.c create mode 100644 flang/test/Lower/RISCV/riscv-vector-bits-vscale-range.f90 delete mode 100644 flang/test/Parser/continuation-before-quote.f90 create mode 100644 flang/test/Parser/continuation-without-ampersand.f90 create mode 100644 libcxx/modules/std.compat.cppm.in create mode 100644 libcxx/modules/std.compat/cassert.inc create mode 100644 libcxx/modules/std.compat/cctype.inc create mode 100644 libcxx/modules/std.compat/cerrno.inc create mode 100644 libcxx/modules/std.compat/cfenv.inc create mode 100644 libcxx/modules/std.compat/cfloat.inc create mode 100644 libcxx/modules/std.compat/cinttypes.inc create mode 100644 libcxx/modules/std.compat/climits.inc create mode 100644 libcxx/modules/std.compat/clocale.inc create mode 100644 libcxx/modules/std.compat/cmath.inc create mode 100644 libcxx/modules/std.compat/csetjmp.inc create mode 100644 libcxx/modules/std.compat/csignal.inc create mode 100644 libcxx/modules/std.compat/cstdarg.inc create mode 100644 libcxx/modules/std.compat/cstddef.inc create mode 100644 libcxx/modules/std.compat/cstdint.inc create mode 100644 libcxx/modules/std.compat/cstdio.inc create mode 100644 libcxx/modules/std.compat/cstdlib.inc create mode 100644 libcxx/modules/std.compat/cstring.inc create mode 100644 libcxx/modules/std.compat/ctime.inc create mode 100644 libcxx/modules/std.compat/cuchar.inc create mode 100644 libcxx/modules/std.compat/cwchar.inc create mode 100644 libcxx/modules/std.compat/cwctype.inc create mode 100644 libcxx/test/libcxx/module_std_compat.gen.py create mode 100644 libcxx/test/libcxx/numerics/rand/rand.req.urng/valid_real_type.verify.cpp create mode 100644 libcxx/test/std/modules/std.compat.pass.cpp rename libcxx/utils/{generate_std_cppm_in.py => generate_libcxx_cppm_in.py} (51%) create mode 100644 libcxx/utils/libcxx/test/modules.py create mode 100644 libcxxabi/test/native/x86_64/lpstart-zero.pass.sh.s delete mode 100644 lld/test/ELF/x86-64-dyn-rel-error2.s create mode 100644 lldb/test/API/driver/quit_speed/Makefile create mode 100644 lldb/test/API/driver/quit_speed/TestQuitWithProcess.py create mode 100644 lldb/test/API/driver/quit_speed/main.c delete mode 100644 lldb/test/API/functionalities/location-list-lookup/main.c create mode 100644 lldb/test/API/functionalities/location-list-lookup/main.cpp create mode 100644 lldb/test/Shell/Commands/command-image-lookup-color.test create mode 100644 llvm/include/llvm-c/LLJITUtils.h create mode 100644 llvm/include/llvm/CodeGen/InterleavedAccess.h create mode 100644 llvm/include/llvm/Support/AMDGPUAddrSpace.h create mode 100644 llvm/lib/ExecutionEngine/Orc/Debugging/LLJITUtilsCBindings.cpp create mode 100644 llvm/lib/Target/BPF/GISel/BPFCallLowering.cpp create mode 100644 llvm/lib/Target/BPF/GISel/BPFCallLowering.h create mode 100644 llvm/lib/Target/BPF/GISel/BPFInstructionSelector.cpp create mode 100644 llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.cpp create mode 100644 llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.h create mode 100644 llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.cpp create mode 100644 llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.h create mode 100644 llvm/lib/Target/BPF/GISel/BPFRegisterBanks.td delete mode 100644 llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types_opaque_ptr.ll create mode 100644 llvm/test/Analysis/LoopAccessAnalysis/num-iters-for-store-load-conflict.ll create mode 100644 llvm/test/Assembler/constant-splat-diagnostics.ll create mode 100644 llvm/test/Assembler/constant-splat.ll create mode 100644 llvm/test/Assembler/summary-parsing-error.ll create mode 100644 llvm/test/CodeGen/AArch64/atomic-oversize.ll create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-prefetch.ll create mode 100644 llvm/test/CodeGen/BPF/GlobalISel/ir-translator-ret.ll create mode 100644 llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll create mode 100644 llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll create mode 100644 llvm/test/CodeGen/LoongArch/lasx/fsqrt.ll create mode 100644 llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fcmp.ll create mode 100644 llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fneg.ll create mode 100644 llvm/test/CodeGen/LoongArch/lasx/ir-instruction/icmp.ll create mode 100644 llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll create mode 100644 llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll create mode 100644 llvm/test/CodeGen/LoongArch/lsx/fsqrt.ll create mode 100644 llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fcmp.ll create mode 100644 llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fneg.ll create mode 100644 llvm/test/CodeGen/LoongArch/lsx/ir-instruction/icmp.ll create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vacopy.ll create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv32.mir create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv64.mir create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vacopy.mir create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv32.mir create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv64.mir create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll create mode 100644 llvm/test/CodeGen/RISCV/macro-fusions-veyron-v1.mir create mode 100644 llvm/test/CodeGen/RISCV/relax-per-target-feature.ll create mode 100644 llvm/test/CodeGen/X86/atomic-nocx16.ll delete mode 100644 llvm/test/CodeGen/X86/atomicf128.ll delete mode 100644 llvm/test/CodeGen/X86/nocx16.ll create mode 100644 llvm/test/CodeGen/X86/pr74736.ll create mode 100644 llvm/test/DebugInfo/assignment-tracking/X86/large-type.ll create mode 100644 llvm/test/Instrumentation/AddressSanitizer/global_metadata_code_model.ll create mode 100644 llvm/test/MC/AMDGPU/gfx12_asm_exp.s create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt create mode 100644 llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt create mode 100644 llvm/test/MC/Disassembler/X86/apx/invpcid.txt create mode 100644 llvm/test/MC/X86/apx/invpcid-att.s create mode 100644 llvm/test/MC/X86/apx/invpcid-intel.s create mode 100644 llvm/test/MachineVerifier/test_g_prefetch.mir create mode 100644 llvm/test/Transforms/AlignmentFromAssumptions/alignment-from-assumptions-track-users.ll create mode 100644 llvm/test/Transforms/InstCombine/ctlz-cttz-shifts.ll create mode 100644 llvm/test/Transforms/LoopUnroll/loop-branch-folding.ll create mode 100644 llvm/test/Transforms/MemCpyOpt/pr75010.ll create mode 100644 llvm/test/Transforms/Reassociate/reassoc-mul-nuw.ll create mode 100644 llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll create mode 100644 llvm/test/tools/llvm-exegesis/X86/latency/snippet-address-annotations-unsupported.s create mode 100644 llvm/test/tools/llvm-exegesis/X86/latency/subprocess-address-annotation.s create mode 100644 llvm/test/tools/llvm-objcopy/regex-error.test create mode 100644 llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s create mode 100644 llvm/test/tools/obj2yaml/XCOFF/aux-symbols.yaml mode change 100644 => 100755 llvm/utils/git/code-format-helper.py create mode 100755 mlir/test/Dialect/SparseTensor/spy_sddmm_bsr.mlir delete mode 100644 mlir/test/Dialect/Tensor/invalid-canonicalize.mlir create mode 100644 mlir/test/IR/custom-print-parse.mlir create mode 100644 mlir/test/IR/invalid-custom-print-parse.mlir create mode 100755 mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir create mode 100644 mlir/test/Target/LLVMIR/Import/nsw_nuw.ll create mode 100644 mlir/test/Target/LLVMIR/nsw_nuw.mlir delete mode 100644 openmp/libomptarget/test/Inputs/empty.c delete mode 100644 openmp/libomptarget/test/offloading/bug60119.c create mode 100644 openmp/libomptarget/test/offloading/bug74582.c create mode 100644 openmp/libomptarget/test/offloading/target_map_for_member_data.cpp diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs index 0e24d6132b4c..74b1e9240d43 100644 --- a/.git-blame-ignore-revs +++ b/.git-blame-ignore-revs @@ -67,3 +67,6 @@ f6d557ee34b6bbdb1dc32f29e34b4a4a8ad35e81 # [libc++] Rename _LIBCPP_INLINE_VISIBILITY to _LIBCPP_HIDE_FROM_ABI 4c198542226223f6a5c5511a1f89b37d15ee10b9 + +# [libc++] Replace uses of _VSTD:: by std:: (#74331) +77a00c0d546cd4aa8311b5b9031ae9ea8cdb050c diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index b9a28edc9daf..8afd2b9ce4a1 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -33,26 +33,42 @@ /lldb/ @JDevlieghere -/mlir/include/mlir/Interfaces/TilingInterface.* @MaheshRavishankar +# Linalg in MLIR. +/mlir/include/mlir/Dialect/Linalg @dcaballe @nicolasvasilache +/mlir/lib/Dialect/Linalg @dcaballe @nicolasvasilache -/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp @MaheshRavishankar -/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp @MaheshRavishankar -/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp @MaheshRavishankar -/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp @MaheshRavishankar -/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp @MaheshRavishankar -/mlir/lib/Interfaces/TilingInterface.* @MaheshRavishankar +# Vector in MLIR. +/mlir/**/*AMX* @dcaballe +/mlir/**/*Neon* @banach-space @dcaballe @nicolasvasilache +/mlir/**/*SME* @banach-space @dcaballe @nicolasvasilache +/mlir/**/*SVE* @banach-space @dcaballe @nicolasvasilache +/mlir/**/*VectorInterfaces* @dcaballe @nicolasvasilache +/mlir/**/*VectorToSCF* @banach-space @dcaballe @nicolasvasilache +/mlir/**/*VectorToLLVM* @banach-space @dcaballe @nicolasvasilache +/mlir/**/*X86Vector* @dcaballe @nicolasvasilache +/mlir/include/mlir/Dialect/Vector @dcaballe @nicolasvasilache +/mlir/lib/Dialect/Vector @dcaballe @nicolasvasilache + +/mlir/include/mlir/Interfaces/TilingInterface.* @MaheshRavishankar @nicolasvasilache + +/mlir/lib/Dialect/Linalg/Transforms/DecomposeLinalgOps.cpp @MaheshRavishankar @nicolasvasilache +/mlir/lib/Dialect/Linalg/Transforms/DropUnitDims.cpp @MaheshRavishankar @nicolasvasilache +/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp @MaheshRavishankar @nicolasvasilache +/mlir/lib/Dialect/MemRef/Transforms/EmulateNarrowType.cpp @MaheshRavishankar @nicolasvasilache +/mlir/lib/Dialect/Vector/Transforms/VectorEmulateNarrowType.cpp @MaheshRavishankar @nicolasvasilache +/mlir/lib/Interfaces/TilingInterface.* @MaheshRavishankar @nicolasvasilache /mlir/**/*EmulateNarrowType* @hanhanW -/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp @hanhanW -/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp @hanhanW -/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp @hanhanW -/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp @hanhanW -/mlir/lib/Dialect/Tensor/Transforms/FoldIntoPackAndUnpackPatterns.cpp @hanhanW -/mlir/lib/Dialect/Vector/Transforms/* @hanhanW +/mlir/lib/Dialect/Linalg/Transforms/DataLayoutPropagation.cpp @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Linalg/Transforms/Transforms.cpp @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Linalg/Transforms/Vectorization.cpp @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Tensor/IR/TensorTilingInterfaceImpl.cpp @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Tensor/Transforms/FoldIntoPackAndUnpackPatterns.cpp @hanhanW @nicolasvasilache +/mlir/lib/Dialect/Vector/Transforms/* @hanhanW @nicolasvasilache # Transform Dialect in MLIR. -/mlir/include/mlir/Dialect/Transform/* @ftynse -/mlir/lib/Dialect/Transform/* @ftynse +/mlir/include/mlir/Dialect/Transform/* @ftynse @nicolasvasilache +/mlir/lib/Dialect/Transform/* @ftynse @nicolasvasilache # SPIR-V in MLIR. /mlir/**/SPIRV/ @antiagainst @kuhar diff --git a/.github/workflows/llvm-project-tests.yml b/.github/workflows/llvm-project-tests.yml index 996cfe41f047..02b1ab75e960 100644 --- a/.github/workflows/llvm-project-tests.yml +++ b/.github/workflows/llvm-project-tests.yml @@ -10,6 +10,11 @@ on: required: false projects: required: false + extra_cmake_args: + required: false + os_list: + required: false + default: '["ubuntu-latest", "windows-2019", "macOS-11"]' workflow_call: inputs: build_target: @@ -20,6 +25,19 @@ on: required: true type: string + extra_cmake_args: + required: false + type: string + + os_list: + required: false + type: string + # Use windows-2019 due to: + # https://developercommunity.visualstudio.com/t/Prev-Issue---with-__assume-isnan-/1597317 + # We're using a specific version of macOS due to: + # https://github.com/actions/virtual-environments/issues/5900 + default: '["ubuntu-latest", "windows-2019", "macOS-11"]' + concurrency: # Skip intermediate builds: always. # Cancel intermediate builds: only if it is a pull request build. @@ -35,14 +53,7 @@ jobs: strategy: fail-fast: false matrix: - os: - - ubuntu-latest - # Use windows-2019 due to: - # https://developercommunity.visualstudio.com/t/Prev-Issue---with-__assume-isnan-/1597317 - - windows-2019 - # We're using a specific version of macOS due to: - # https://github.com/actions/virtual-environments/issues/5900 - - macOS-11 + os: ${{ fromJSON(inputs.os_list) }} steps: - name: Setup Windows if: startsWith(matrix.os, 'windows') @@ -85,7 +96,7 @@ jobs: # This should be a no-op for non-mac OSes PKG_CONFIG_PATH: /usr/local/Homebrew/Library/Homebrew/os/mac/pkgconfig//12 with: - cmake_args: '-GNinja -DLLVM_ENABLE_PROJECTS="${{ inputs.projects }}" -DCMAKE_BUILD_TYPE=Release -DLLDB_INCLUDE_TESTS=OFF -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache' + cmake_args: '-GNinja -DLLVM_ENABLE_PROJECTS="${{ inputs.projects }}" -DCMAKE_BUILD_TYPE=Release -DLLVM_ENABLE_ASSERTIONS=ON -DLLDB_INCLUDE_TESTS=OFF -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache ${{ inputs.extra_cmake_args }}' build_target: '${{ inputs.build_target }}' - name: Build and Test libclc diff --git a/.github/workflows/spirv-tests.yml b/.github/workflows/spirv-tests.yml new file mode 100644 index 000000000000..0e1d0caca410 --- /dev/null +++ b/.github/workflows/spirv-tests.yml @@ -0,0 +1,29 @@ +name: SPIR-V Tests + +permissions: + contents: read + +on: + workflow_dispatch: + pull_request: + paths: + - 'llvm/lib/Target/SPIRV/**' + - 'llvm/test/CodeGen/SPIRV/**' + - '.github/workflows/spirv-tests.yml' + +concurrency: + # Skip intermediate builds: always. + # Cancel intermediate builds: only if it is a pull request build. + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ startsWith(github.ref, 'refs/pull/') }} + +jobs: + check_spirv: + if: github.repository_owner == 'llvm' + name: Test SPIR-V + uses: ./.github/workflows/llvm-project-tests.yml + with: + build_target: check-llvm-codegen-spirv + projects: + extra_cmake_args: '-DLLVM_TARGETS_TO_BUILD="" -DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD="SPIRV"' + os_list: '["ubuntu-latest"]' diff --git a/.mailmap b/.mailmap index f4d7ffe30138..d5464cf9e815 100644 --- a/.mailmap +++ b/.mailmap @@ -33,6 +33,8 @@ +Jianjian GUAN +Jianjian GUAN Jon Roelofs Jon Roelofs Jonathan Thackray diff --git a/bolt/include/bolt/Core/BinaryFunction.h b/bolt/include/bolt/Core/BinaryFunction.h index 182d5ff049c3..3a1eae3311bd 100644 --- a/bolt/include/bolt/Core/BinaryFunction.h +++ b/bolt/include/bolt/Core/BinaryFunction.h @@ -75,6 +75,14 @@ enum IndirectCallPromotionType : char { ICP_ALL /// Perform ICP on calls and jump tables. }; +/// Hash functions supported for BF/BB hashing. +enum class HashFunction : char { + StdHash, /// std::hash, implementation is platform-dependent. Provided for + /// backwards compatibility. + XXH3, /// llvm::xxh3_64bits, the default. + Default = XXH3, +}; + /// Information on a single indirect call to a particular callee. struct IndirectCallProfile { MCSymbol *Symbol; @@ -2234,18 +2242,21 @@ public: /// /// If \p UseDFS is set, process basic blocks in DFS order. Otherwise, use /// the existing layout order. + /// \p HashFunction specifies which function is used for BF hashing. /// /// By default, instruction operands are ignored while calculating the hash. /// The caller can change this via passing \p OperandHashFunc function. /// The return result of this function will be mixed with internal hash. size_t computeHash( - bool UseDFS = false, + bool UseDFS = false, HashFunction HashFunction = HashFunction::Default, OperandHashFuncTy OperandHashFunc = [](const MCOperand &) { return std::string(); }) const; /// Compute hash values for each block of the function. - void computeBlockHashes() const; + /// \p HashFunction specifies which function is used for BB hashing. + void + computeBlockHashes(HashFunction HashFunction = HashFunction::Default) const; void setDWARFUnit(DWARFUnit *Unit) { DwarfUnit = Unit; } diff --git a/bolt/include/bolt/Profile/ProfileYAMLMapping.h b/bolt/include/bolt/Profile/ProfileYAMLMapping.h index 2218a167a74e..548b528ae2d6 100644 --- a/bolt/include/bolt/Profile/ProfileYAMLMapping.h +++ b/bolt/include/bolt/Profile/ProfileYAMLMapping.h @@ -178,6 +178,14 @@ template <> struct ScalarBitSetTraits { } }; +template <> struct ScalarEnumerationTraits { + using HashFunction = llvm::bolt::HashFunction; + static void enumeration(IO &io, HashFunction &value) { + io.enumCase(value, "std-hash", HashFunction::StdHash); + io.enumCase(value, "xxh3", HashFunction::XXH3); + } +}; + namespace bolt { struct BinaryProfileHeader { uint32_t Version{1}; @@ -188,6 +196,7 @@ struct BinaryProfileHeader { std::string Origin; // How the profile was obtained. std::string EventNames; // Events used for sample profile. bool IsDFSOrder{true}; // Whether using DFS block order in function profile + llvm::bolt::HashFunction HashFunction; // Hash used for BB/BF hashing }; } // end namespace bolt @@ -200,6 +209,8 @@ template <> struct MappingTraits { YamlIO.mapOptional("profile-origin", Header.Origin); YamlIO.mapOptional("profile-events", Header.EventNames); YamlIO.mapOptional("dfs-order", Header.IsDFSOrder); + YamlIO.mapOptional("hash-func", Header.HashFunction, + llvm::bolt::HashFunction::StdHash); } }; diff --git a/bolt/lib/Core/BinaryFunction.cpp b/bolt/lib/Core/BinaryFunction.cpp index be033cf07668..0ac47a53a446 100644 --- a/bolt/lib/Core/BinaryFunction.cpp +++ b/bolt/lib/Core/BinaryFunction.cpp @@ -3633,7 +3633,7 @@ BinaryFunction::BasicBlockListType BinaryFunction::dfs() const { return DFS; } -size_t BinaryFunction::computeHash(bool UseDFS, +size_t BinaryFunction::computeHash(bool UseDFS, HashFunction HashFunction, OperandHashFuncTy OperandHashFunc) const { if (size() == 0) return 0; @@ -3652,7 +3652,13 @@ size_t BinaryFunction::computeHash(bool UseDFS, for (const BinaryBasicBlock *BB : Order) HashString.append(hashBlock(BC, *BB, OperandHashFunc)); - return Hash = llvm::xxh3_64bits(HashString); + switch (HashFunction) { + case HashFunction::StdHash: + return Hash = std::hash{}(HashString); + case HashFunction::XXH3: + return Hash = llvm::xxh3_64bits(HashString); + } + llvm_unreachable("Unhandled HashFunction"); } void BinaryFunction::insertBasicBlocks( diff --git a/bolt/lib/Core/BinaryFunctionProfile.cpp b/bolt/lib/Core/BinaryFunctionProfile.cpp index 0d705cd82f5d..55ebe5fc900e 100644 --- a/bolt/lib/Core/BinaryFunctionProfile.cpp +++ b/bolt/lib/Core/BinaryFunctionProfile.cpp @@ -225,6 +225,7 @@ void BinaryFunction::mergeProfileDataInto(BinaryFunction &BF) const { for (const BinaryBasicBlock *BBSucc : BB->successors()) { (void)BBSucc; assert(getIndex(BBSucc) == BF.getIndex(*BBMergeSI)); + (void)BBMergeSI; // At this point no branch count should be set to COUNT_NO_PROFILE. assert(BII->Count != BinaryBasicBlock::COUNT_NO_PROFILE && diff --git a/bolt/lib/Core/Exceptions.cpp b/bolt/lib/Core/Exceptions.cpp index 993f3a7770aa..ab1885f6bb58 100644 --- a/bolt/lib/Core/Exceptions.cpp +++ b/bolt/lib/Core/Exceptions.cpp @@ -108,7 +108,8 @@ void BinaryFunction::parseLSDA(ArrayRef LSDASectionData, DWARFDataExtractor Data( StringRef(reinterpret_cast(LSDASectionData.data()), LSDASectionData.size()), - BC.DwCtx->getDWARFObj().isLittleEndian(), 8); + BC.DwCtx->getDWARFObj().isLittleEndian(), + BC.DwCtx->getDWARFObj().getAddressSize()); uint64_t Offset = getLSDAAddress() - LSDASectionAddress; assert(Data.isValidOffset(Offset) && "wrong LSDA address"); diff --git a/bolt/lib/Passes/IdenticalCodeFolding.cpp b/bolt/lib/Passes/IdenticalCodeFolding.cpp index b4ec89ca8fd7..dfbc72e48e5d 100644 --- a/bolt/lib/Passes/IdenticalCodeFolding.cpp +++ b/bolt/lib/Passes/IdenticalCodeFolding.cpp @@ -360,9 +360,9 @@ void IdenticalCodeFolding::runOnFunctions(BinaryContext &BC) { // Pre-compute hash before pushing into hashtable. // Hash instruction operands to minimize hash collisions. - BF.computeHash(opts::ICFUseDFS, [&BC](const MCOperand &Op) { - return hashInstOperand(BC, Op); - }); + BF.computeHash( + opts::ICFUseDFS, HashFunction::Default, + [&BC](const MCOperand &Op) { return hashInstOperand(BC, Op); }); }; ParallelUtilities::PredicateTy SkipFunc = [&](const BinaryFunction &BF) { diff --git a/bolt/lib/Passes/LongJmp.cpp b/bolt/lib/Passes/LongJmp.cpp index a81689bc3746..ded0db2cd30b 100644 --- a/bolt/lib/Passes/LongJmp.cpp +++ b/bolt/lib/Passes/LongJmp.cpp @@ -202,10 +202,23 @@ LongJmpPass::replaceTargetWithStub(BinaryBasicBlock &BB, MCInst &Inst, } } else if (LocalStubsIter != Stubs.end() && LocalStubsIter->second.count(TgtBB)) { - // If we are replacing a local stub (because it is now out of range), - // use its target instead of creating a stub to jump to another stub + // The TgtBB and TgtSym now are the local out-of-range stub and its label. + // So, we are attempting to restore BB to its previous state without using + // this stub. TgtSym = BC.MIB->getTargetSymbol(*TgtBB->begin()); - TgtBB = BB.getSuccessor(TgtSym, BI); + assert(TgtSym && + "First instruction is expected to contain a target symbol."); + BinaryBasicBlock *TgtBBSucc = TgtBB->getSuccessor(TgtSym, BI); + + // TgtBB might have no successor. e.g. a stub for a function call. + if (TgtBBSucc) { + BB.replaceSuccessor(TgtBB, TgtBBSucc, BI.Count, BI.MispredictedCount); + assert(TgtBB->getExecutionCount() >= BI.Count && + "At least equal or greater than the branch count."); + TgtBB->setExecutionCount(TgtBB->getExecutionCount() - BI.Count); + } + + TgtBB = TgtBBSucc; } BinaryBasicBlock *StubBB = lookupLocalStub(BB, Inst, TgtSym, DotAddress); diff --git a/bolt/lib/Passes/VeneerElimination.cpp b/bolt/lib/Passes/VeneerElimination.cpp index eadbfc17fb97..929c7360b7ff 100644 --- a/bolt/lib/Passes/VeneerElimination.cpp +++ b/bolt/lib/Passes/VeneerElimination.cpp @@ -89,6 +89,7 @@ void VeneerElimination::runOnFunctions(BinaryContext &BC) { LLVM_DEBUG( dbgs() << "BOLT-INFO: number of linker-inserted veneers call sites: " << VeneerCallers << "\n"); + (void)VeneerCallers; } } // namespace bolt diff --git a/bolt/lib/Profile/StaleProfileMatching.cpp b/bolt/lib/Profile/StaleProfileMatching.cpp index 6fb6f380f71e..26180f132147 100644 --- a/bolt/lib/Profile/StaleProfileMatching.cpp +++ b/bolt/lib/Profile/StaleProfileMatching.cpp @@ -225,7 +225,7 @@ private: std::unordered_map> OpHashToBlocks; }; -void BinaryFunction::computeBlockHashes() const { +void BinaryFunction::computeBlockHashes(HashFunction HashFunction) const { if (size() == 0) return; @@ -241,12 +241,26 @@ void BinaryFunction::computeBlockHashes() const { // Hashing complete instructions. std::string InstrHashStr = hashBlock( BC, *BB, [&](const MCOperand &Op) { return hashInstOperand(BC, Op); }); - uint64_t InstrHash = llvm::xxh3_64bits(InstrHashStr); - BlendedHashes[I].InstrHash = (uint16_t)InstrHash; + if (HashFunction == HashFunction::StdHash) { + uint64_t InstrHash = std::hash{}(InstrHashStr); + BlendedHashes[I].InstrHash = (uint16_t)hash_value(InstrHash); + } else if (HashFunction == HashFunction::XXH3) { + uint64_t InstrHash = llvm::xxh3_64bits(InstrHashStr); + BlendedHashes[I].InstrHash = (uint16_t)InstrHash; + } else { + llvm_unreachable("Unhandled HashFunction"); + } // Hashing opcodes. std::string OpcodeHashStr = hashBlockLoose(BC, *BB); - OpcodeHashes[I] = llvm::xxh3_64bits(OpcodeHashStr); - BlendedHashes[I].OpcodeHash = (uint16_t)OpcodeHashes[I]; + if (HashFunction == HashFunction::StdHash) { + OpcodeHashes[I] = std::hash{}(OpcodeHashStr); + BlendedHashes[I].OpcodeHash = (uint16_t)hash_value(OpcodeHashes[I]); + } else if (HashFunction == HashFunction::XXH3) { + OpcodeHashes[I] = llvm::xxh3_64bits(OpcodeHashStr); + BlendedHashes[I].OpcodeHash = (uint16_t)OpcodeHashes[I]; + } else { + llvm_unreachable("Unhandled HashFunction"); + } } // Initialize neighbor hash. @@ -258,7 +272,12 @@ void BinaryFunction::computeBlockHashes() const { uint64_t SuccHash = OpcodeHashes[SuccBB->getIndex()]; Hash = hashing::detail::hash_16_bytes(Hash, SuccHash); } - BlendedHashes[I].SuccHash = (uint8_t)Hash; + if (HashFunction == HashFunction::StdHash) { + // Compatibility with old behavior. + BlendedHashes[I].SuccHash = (uint8_t)hash_value(Hash); + } else { + BlendedHashes[I].SuccHash = (uint8_t)Hash; + } // Append hashes of predecessors. Hash = 0; @@ -266,7 +285,12 @@ void BinaryFunction::computeBlockHashes() const { uint64_t PredHash = OpcodeHashes[PredBB->getIndex()]; Hash = hashing::detail::hash_16_bytes(Hash, PredHash); } - BlendedHashes[I].PredHash = (uint8_t)Hash; + if (HashFunction == HashFunction::StdHash) { + // Compatibility with old behavior. + BlendedHashes[I].PredHash = (uint8_t)hash_value(Hash); + } else { + BlendedHashes[I].PredHash = (uint8_t)Hash; + } } // Assign hashes. @@ -682,7 +706,7 @@ bool YAMLProfileReader::inferStaleProfile( << "\"" << BF.getPrintName() << "\"\n"); // Make sure that block hashes are up to date. - BF.computeBlockHashes(); + BF.computeBlockHashes(YamlBP.Header.HashFunction); const BinaryFunction::BasicBlockOrderType BlockOrder( BF.getLayout().block_begin(), BF.getLayout().block_end()); diff --git a/bolt/lib/Profile/YAMLProfileReader.cpp b/bolt/lib/Profile/YAMLProfileReader.cpp index 079cb352d36e..ade562ef6fb1 100644 --- a/bolt/lib/Profile/YAMLProfileReader.cpp +++ b/bolt/lib/Profile/YAMLProfileReader.cpp @@ -83,6 +83,7 @@ bool YAMLProfileReader::parseFunctionProfile( BinaryContext &BC = BF.getBinaryContext(); const bool IsDFSOrder = YamlBP.Header.IsDFSOrder; + const HashFunction HashFunction = YamlBP.Header.HashFunction; bool ProfileMatched = true; uint64_t MismatchedBlocks = 0; uint64_t MismatchedCalls = 0; @@ -98,7 +99,8 @@ bool YAMLProfileReader::parseFunctionProfile( FuncRawBranchCount += YamlSI.Count; BF.setRawBranchCount(FuncRawBranchCount); - if (!opts::IgnoreHash && YamlBF.Hash != BF.computeHash(IsDFSOrder)) { + if (!opts::IgnoreHash && + YamlBF.Hash != BF.computeHash(IsDFSOrder, HashFunction)) { if (opts::Verbosity >= 1) errs() << "BOLT-WARNING: function hash mismatch\n"; ProfileMatched = false; @@ -326,6 +328,17 @@ bool YAMLProfileReader::mayHaveProfileData(const BinaryFunction &BF) { } Error YAMLProfileReader::readProfile(BinaryContext &BC) { + if (opts::Verbosity >= 1) { + outs() << "BOLT-INFO: YAML profile with hash: "; + switch (YamlBP.Header.HashFunction) { + case HashFunction::StdHash: + outs() << "std::hash\n"; + break; + case HashFunction::XXH3: + outs() << "xxh3\n"; + break; + } + } YamlProfileToFunction.resize(YamlBP.Functions.size() + 1); auto profileMatches = [](const yaml::bolt::BinaryFunctionProfile &Profile, @@ -348,7 +361,8 @@ Error YAMLProfileReader::readProfile(BinaryContext &BC) { // Recompute hash once per function. if (!opts::IgnoreHash) - Function.computeHash(YamlBP.Header.IsDFSOrder); + Function.computeHash(YamlBP.Header.IsDFSOrder, + YamlBP.Header.HashFunction); if (profileMatches(YamlBF, Function)) matchProfileToFunction(YamlBF, Function); diff --git a/bolt/lib/Profile/YAMLProfileWriter.cpp b/bolt/lib/Profile/YAMLProfileWriter.cpp index 3326d1d8f559..dffd851a1d6f 100644 --- a/bolt/lib/Profile/YAMLProfileWriter.cpp +++ b/bolt/lib/Profile/YAMLProfileWriter.cpp @@ -189,6 +189,7 @@ std::error_code YAMLProfileWriter::writeProfile(const RewriteInstance &RI) { BP.Header.Id = BuildID ? std::string(*BuildID) : ""; BP.Header.Origin = std::string(RI.getProfileReader()->getReaderName()); BP.Header.IsDFSOrder = opts::ProfileUseDFS; + BP.Header.HashFunction = HashFunction::Default; StringSet<> EventNames = RI.getProfileReader()->getEventNames(); if (!EventNames.empty()) { diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp index 8cda0b7fcca9..1e8ca569682f 100644 --- a/bolt/lib/Rewrite/RewriteInstance.cpp +++ b/bolt/lib/Rewrite/RewriteInstance.cpp @@ -623,7 +623,9 @@ void RewriteInstance::parseBuildID() { // Reading notes section (see Portable Formats Specification, Version 1.1, // pg 2-5, section "Note Section"). - DataExtractor DE = DataExtractor(Buf, true, 8); + DataExtractor DE = + DataExtractor(Buf, + /*IsLittleEndian=*/true, InputFile->getBytesInAddress()); uint64_t Offset = 0; if (!DE.isValidOffset(Offset)) return; diff --git a/bolt/test/X86/Inputs/blarge_profile_stale.std-hash.yaml b/bolt/test/X86/Inputs/blarge_profile_stale.std-hash.yaml new file mode 100644 index 000000000000..d520a0d242bf --- /dev/null +++ b/bolt/test/X86/Inputs/blarge_profile_stale.std-hash.yaml @@ -0,0 +1,56 @@ +--- +header: + profile-version: 1 + binary-name: 'reader-yaml.test.tmp.exe' + binary-build-id: '' + profile-flags: [ lbr ] + profile-origin: branch profile reader + profile-events: '' + dfs-order: false +functions: + - name: SolveCubic + fid: 6 + hash: 0xC6E9098E973BBE19 + exec: 151 + nblocks: 18 + blocks: + - bid: 0 + insns: 43 + hash: 0xed4db287e71c0000 + exec: 151 + succ: [ { bid: 1, cnt: 151, mis: 2 }, { bid: 7, cnt: 0 } ] + - bid: 1 + insns: 7 + hash: 0x39330000e4560088 + succ: [ { bid: 13, cnt: 151 }, { bid: 2, cnt: 0 } ] + - bid: 13 + insns: 26 + hash: 0xa9700000fe202a7 + succ: [ { bid: 3, cnt: 89 }, { bid: 2, cnt: 10 } ] + - bid: 3 + insns: 9 + hash: 0x62391dad18a700a0 + succ: [ { bid: 5, cnt: 151 } ] + - bid: 5 + insns: 9 + hash: 0x4d906d19ecec0111 + - name: usqrt + fid: 7 + hash: 0x8B62B1F9AD81EA35 + exec: 20 + nblocks: 6 + blocks: + - bid: 0 + insns: 4 + hash: 0x1111111111111111 + exec: 20 + succ: [ { bid: 1, cnt: 0 } ] + - bid: 1 + insns: 9 + hash: 0x27e43a5e10cd0010 + succ: [ { bid: 3, cnt: 320, mis: 171 }, { bid: 2, cnt: 0 } ] + - bid: 3 + insns: 2 + hash: 0x4db935b6471e0039 + succ: [ { bid: 1, cnt: 300, mis: 33 }, { bid: 4, cnt: 20 } ] +... diff --git a/bolt/test/X86/Inputs/blarge_profile_stale.yaml b/bolt/test/X86/Inputs/blarge_profile_stale.yaml index 43b75c99656f..ac46b37b56a1 100644 --- a/bolt/test/X86/Inputs/blarge_profile_stale.yaml +++ b/bolt/test/X86/Inputs/blarge_profile_stale.yaml @@ -7,6 +7,7 @@ header: profile-origin: branch profile reader profile-events: '' dfs-order: false + hash-func: xxh3 functions: - name: SolveCubic fid: 6 diff --git a/bolt/test/X86/reader-stale-yaml-std.test b/bolt/test/X86/reader-stale-yaml-std.test new file mode 100644 index 000000000000..e0b6ca0645e1 --- /dev/null +++ b/bolt/test/X86/reader-stale-yaml-std.test @@ -0,0 +1,68 @@ +# This script checks that YamlProfileReader in llvm-bolt is reading data +# correctly and stale data is corrected by profile inference. + +RUN: yaml2obj %p/Inputs/blarge.yaml &> %t.exe +RUN: llvm-bolt %t.exe -o %t.null -b %p/Inputs/blarge_profile_stale.std-hash.yaml \ +RUN: --print-cfg --print-only=usqrt,SolveCubic --infer-stale-profile=1 -v=1 \ +RUN: 2>&1 | FileCheck %s + +# Verify that yaml reader works as expected. +CHECK: pre-processing profile using YAML profile reader +CHECK: BOLT-INFO: YAML profile with hash: std::hash + +# Function "SolveCubic" has stale profile, since there is one jump in the +# profile (from bid=13 to bid=2) which is not in the CFG in the binary. The test +# verifies that the inference is able to match two blocks (bid=1 and bid=13) +# using "loose" hashes and then correctly propagate the counts. + +CHECK: Binary Function "SolveCubic" after building cfg { +CHECK: State : CFG constructed +CHECK: Address : 0x400e00 +CHECK: Size : 0x368 +CHECK: Section : .text +CHECK: IsSimple : 1 +CHECK: BB Count : 18 +CHECK: Exec Count : 151 +CHECK: Branch Count: 552 +CHECK: } +# Verify block counts. +CHECK: .LBB00 (43 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB07:]] (mispreds: 0, count: 0), .LFT[[#BB01:]] (mispreds: 0, count: 151) +CHECK: .LFT[[#BB01:]] (5 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB013:]] (mispreds: 0, count: 151), .LFT[[#BB02:]] (mispreds: 0, count: 0) +CHECK: .Ltmp[[#BB03:]] (26 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB05:]] (mispreds: 0, count: 151), .LFT[[#BB04:]] (mispreds: 0, count: 0) +CHECK: .Ltmp[[#BB05:]] (9 instructions, align : 1) +CHECK: .Ltmp[[#BB013:]] (12 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB03:]] (mispreds: 0, count: 151) +CHECK: End of Function "SolveCubic" + +# Function "usqrt" has stale profile, since the number of blocks in the profile +# (nblocks=6) does not match the size of the CFG in the binary. The entry +# block (bid=0) has an incorrect (missing) count, which should be inferred by +# the algorithm. + +CHECK: Binary Function "usqrt" after building cfg { +CHECK: State : CFG constructed +CHECK: Address : 0x401170 +CHECK: Size : 0x43 +CHECK: Section : .text +CHECK: IsSimple : 1 +CHECK: BB Count : 5 +CHECK: Exec Count : 20 +CHECK: Branch Count: 640 +CHECK: } +# Verify block counts. +CHECK: .LBB01 (4 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB113:]] (mispreds: 0, count: 20) +CHECK: .Ltmp[[#BB113:]] (9 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB112:]] (mispreds: 0, count: 320), .LFT[[#BB10:]] (mispreds: 0, count: 0) +CHECK: .LFT[[#BB10:]] (2 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB112:]] (mispreds: 0, count: 0) +CHECK: .Ltmp[[#BB112:]] (2 instructions, align : 1) +CHECK: Successors: .Ltmp[[#BB113:]] (mispreds: 0, count: 300), .LFT[[#BB11:]] (mispreds: 0, count: 20) +CHECK: .LFT[[#BB11:]] (2 instructions, align : 1) +CHECK: End of Function "usqrt" +# Check the overall inference stats. +CHECK: 2 out of 7 functions in the binary (28.6%) have non-empty execution profile +CHECK: inferred profile for 2 (100.00% of profiled, 100.00% of stale) functions responsible for {{.*}} samples ({{.*}} out of {{.*}}) diff --git a/clang-tools-extra/clang-doc/Mapper.cpp b/clang-tools-extra/clang-doc/Mapper.cpp index 5264417748a1..bb8b7952980a 100644 --- a/clang-tools-extra/clang-doc/Mapper.cpp +++ b/clang-tools-extra/clang-doc/Mapper.cpp @@ -103,7 +103,7 @@ llvm::SmallString<128> MapASTVisitor::getFile(const NamedDecl *D, .getPresumedLoc(D->getBeginLoc()) .getFilename()); IsFileInRootDir = false; - if (RootDir.empty() || !File.startswith(RootDir)) + if (RootDir.empty() || !File.starts_with(RootDir)) return File; IsFileInRootDir = true; llvm::SmallString<128> Prefix(RootDir); diff --git a/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp b/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp index 52cc2e6569b0..0b1e9f59e1a7 100644 --- a/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp +++ b/clang-tools-extra/clang-tidy/ExpandModularHeadersPPCallbacks.cpp @@ -171,7 +171,7 @@ void ExpandModularHeadersPPCallbacks::InclusionDirective( if (Imported) { serialization::ModuleFile *MF = Compiler.getASTReader()->getModuleManager().lookup( - Imported->getASTFile()); + *Imported->getASTFile()); handleModuleFile(MF); } parseToLocation(DirectiveLoc); diff --git a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp index 6bc9f2dd367d..05012c7df6a9 100644 --- a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp +++ b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.cpp @@ -133,6 +133,21 @@ UnusedReturnValueCheck::UnusedReturnValueCheck(llvm::StringRef Name, "::boost::system::error_code"))), AllowCastToVoid(Options.get("AllowCastToVoid", false)) {} +UnusedReturnValueCheck::UnusedReturnValueCheck(llvm::StringRef Name, + ClangTidyContext *Context, + std::string CheckedFunctions) + : UnusedReturnValueCheck(Name, Context, std::move(CheckedFunctions), {}, + false) {} + +UnusedReturnValueCheck::UnusedReturnValueCheck( + llvm::StringRef Name, ClangTidyContext *Context, + std::string CheckedFunctions, std::vector CheckedReturnTypes, + bool AllowCastToVoid) + : ClangTidyCheck(Name, Context), + CheckedFunctions(std::move(CheckedFunctions)), + CheckedReturnTypes(std::move(CheckedReturnTypes)), + AllowCastToVoid(AllowCastToVoid) {} + void UnusedReturnValueCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { Options.store(Opts, "CheckedFunctions", CheckedFunctions); Options.store(Opts, "CheckedReturnTypes", diff --git a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h index b4356f8379fd..ab2cc691b894 100644 --- a/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h +++ b/clang-tools-extra/clang-tidy/bugprone/UnusedReturnValueCheck.h @@ -31,7 +31,15 @@ public: private: std::string CheckedFunctions; const std::vector CheckedReturnTypes; - const bool AllowCastToVoid; + +protected: + UnusedReturnValueCheck(StringRef Name, ClangTidyContext *Context, + std::string CheckedFunctions); + UnusedReturnValueCheck(StringRef Name, ClangTidyContext *Context, + std::string CheckedFunctions, + std::vector CheckedReturnTypes, + bool AllowCastToVoid); + bool AllowCastToVoid; }; } // namespace clang::tidy::bugprone diff --git a/clang-tools-extra/clang-tidy/hicpp/CMakeLists.txt b/clang-tools-extra/clang-tidy/hicpp/CMakeLists.txt index d12ca275d396..132fbaccccf8 100644 --- a/clang-tools-extra/clang-tidy/hicpp/CMakeLists.txt +++ b/clang-tools-extra/clang-tidy/hicpp/CMakeLists.txt @@ -6,6 +6,7 @@ set(LLVM_LINK_COMPONENTS add_clang_library(clangTidyHICPPModule ExceptionBaseclassCheck.cpp HICPPTidyModule.cpp + IgnoredRemoveResultCheck.cpp MultiwayPathsCoveredCheck.cpp NoAssemblerCheck.cpp SignedBitwiseCheck.cpp diff --git a/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp b/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp index 374979687712..daa9f398a740 100644 --- a/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp +++ b/clang-tools-extra/clang-tidy/hicpp/HICPPTidyModule.cpp @@ -37,6 +37,7 @@ #include "../readability/NamedParameterCheck.h" #include "../readability/UppercaseLiteralSuffixCheck.h" #include "ExceptionBaseclassCheck.h" +#include "IgnoredRemoveResultCheck.h" #include "MultiwayPathsCoveredCheck.h" #include "NoAssemblerCheck.h" #include "SignedBitwiseCheck.h" @@ -57,6 +58,8 @@ public: "hicpp-deprecated-headers"); CheckFactories.registerCheck( "hicpp-exception-baseclass"); + CheckFactories.registerCheck( + "hicpp-ignored-remove-result"); CheckFactories.registerCheck( "hicpp-multiway-paths-covered"); CheckFactories.registerCheck("hicpp-signed-bitwise"); diff --git a/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.cpp b/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.cpp new file mode 100644 index 000000000000..3410559d435f --- /dev/null +++ b/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.cpp @@ -0,0 +1,28 @@ +//===--- IgnoredRemoveResultCheck.cpp - clang-tidy ------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "IgnoredRemoveResultCheck.h" + +namespace clang::tidy::hicpp { + +IgnoredRemoveResultCheck::IgnoredRemoveResultCheck(llvm::StringRef Name, + ClangTidyContext *Context) + : UnusedReturnValueCheck(Name, Context, + "::std::remove;" + "::std::remove_if;" + "::std::unique") { + // The constructor for ClangTidyCheck needs to have been called + // before we can access options via Options.get(). + AllowCastToVoid = Options.get("AllowCastToVoid", true); +} + +void IgnoredRemoveResultCheck::storeOptions(ClangTidyOptions::OptionMap &Opts) { + Options.store(Opts, "AllowCastToVoid", AllowCastToVoid); +} + +} // namespace clang::tidy::hicpp diff --git a/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.h b/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.h new file mode 100644 index 000000000000..48354c34a858 --- /dev/null +++ b/clang-tools-extra/clang-tidy/hicpp/IgnoredRemoveResultCheck.h @@ -0,0 +1,29 @@ +//===--- IgnoredRemoveResultCheck.h - clang-tidy ----------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_IGNOREDREMOVERESULTCHECK_H +#define LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_IGNOREDREMOVERESULTCHECK_H + +#include "../bugprone/UnusedReturnValueCheck.h" + +namespace clang::tidy::hicpp { + +/// Ensure that the result of std::remove, std::remove_if and std::unique +/// are not ignored according to rule 17.5.1. +/// +/// For the user-facing documentation see: +/// http://clang.llvm.org/extra/clang-tidy/checks/hicpp/ignored-remove-result.html +class IgnoredRemoveResultCheck : public bugprone::UnusedReturnValueCheck { +public: + IgnoredRemoveResultCheck(StringRef Name, ClangTidyContext *Context); + void storeOptions(ClangTidyOptions::OptionMap &Opts) override; +}; + +} // namespace clang::tidy::hicpp + +#endif // LLVM_CLANG_TOOLS_EXTRA_CLANG_TIDY_HICPP_IGNOREDREMOVERESULTCHECK_H diff --git a/clang-tools-extra/clang-tidy/misc/IncludeCleanerCheck.cpp b/clang-tools-extra/clang-tidy/misc/IncludeCleanerCheck.cpp index e336ba1ee1fa..5ae6caedb7f4 100644 --- a/clang-tools-extra/clang-tidy/misc/IncludeCleanerCheck.cpp +++ b/clang-tools-extra/clang-tidy/misc/IncludeCleanerCheck.cpp @@ -124,7 +124,7 @@ void IncludeCleanerCheck::check(const MatchFinder::MatchResult &Result) { MainFileDecls.push_back(D); } llvm::DenseSet SeenSymbols; - const DirectoryEntry *ResourceDir = + OptionalDirectoryEntryRef ResourceDir = PP->getHeaderSearchInfo().getModuleMap().getBuiltinDir(); // FIXME: Find a way to have less code duplication between include-cleaner // analysis implementation and the below code. diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp index 990e20400fbf..dfe12c5b6007 100644 --- a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp +++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.cpp @@ -15,6 +15,7 @@ #include "clang/AST/Decl.h" #include "clang/Basic/Diagnostic.h" #include +#include namespace clang::tidy::performance { namespace { @@ -263,19 +264,25 @@ void UnnecessaryCopyInitialization::registerMatchers(MatchFinder *Finder) { void UnnecessaryCopyInitialization::check( const MatchFinder::MatchResult &Result) { - const auto *NewVar = Result.Nodes.getNodeAs("newVarDecl"); + const auto &NewVar = *Result.Nodes.getNodeAs("newVarDecl"); + const auto &BlockStmt = *Result.Nodes.getNodeAs("blockStmt"); + const auto &VarDeclStmt = *Result.Nodes.getNodeAs("declStmt"); + // Do not propose fixes if the DeclStmt has multiple VarDecls or in + // macros since we cannot place them correctly. + const bool IssueFix = + VarDeclStmt.isSingleDecl() && !NewVar.getLocation().isMacroID(); + const bool IsVarUnused = isVariableUnused(NewVar, BlockStmt, *Result.Context); + const bool IsVarOnlyUsedAsConst = + isOnlyUsedAsConst(NewVar, BlockStmt, *Result.Context); + const CheckContext Context{ + NewVar, BlockStmt, VarDeclStmt, *Result.Context, + IssueFix, IsVarUnused, IsVarOnlyUsedAsConst}; const auto *OldVar = Result.Nodes.getNodeAs(OldVarDeclId); const auto *ObjectArg = Result.Nodes.getNodeAs(ObjectArgId); - const auto *BlockStmt = Result.Nodes.getNodeAs("blockStmt"); const auto *CtorCall = Result.Nodes.getNodeAs("ctorCall"); - const auto *Stmt = Result.Nodes.getNodeAs("declStmt"); TraversalKindScope RAII(*Result.Context, TK_AsIs); - // Do not propose fixes if the DeclStmt has multiple VarDecls or in macros - // since we cannot place them correctly. - bool IssueFix = Stmt->isSingleDecl() && !NewVar->getLocation().isMacroID(); - // A constructor that looks like T(const T& t, bool arg = false) counts as a // copy only when it is called with default arguments for the arguments after // the first. @@ -289,74 +296,71 @@ void UnnecessaryCopyInitialization::check( // instantiations where the types differ and rely on implicit conversion would // no longer compile if we switched to a reference. if (differentReplacedTemplateParams( - NewVar->getType(), constructorArgumentType(OldVar, Result.Nodes), + Context.Var.getType(), constructorArgumentType(OldVar, Result.Nodes), *Result.Context)) return; if (OldVar == nullptr) { - handleCopyFromMethodReturn(*NewVar, *BlockStmt, *Stmt, IssueFix, ObjectArg, - *Result.Context); + // `auto NewVar = functionCall();` + handleCopyFromMethodReturn(Context, ObjectArg); } else { - handleCopyFromLocalVar(*NewVar, *OldVar, *BlockStmt, *Stmt, IssueFix, - *Result.Context); + // `auto NewVar = OldVar;` + handleCopyFromLocalVar(Context, *OldVar); } } void UnnecessaryCopyInitialization::handleCopyFromMethodReturn( - const VarDecl &Var, const Stmt &BlockStmt, const DeclStmt &Stmt, - bool IssueFix, const VarDecl *ObjectArg, ASTContext &Context) { - bool IsConstQualified = Var.getType().isConstQualified(); - if (!IsConstQualified && !isOnlyUsedAsConst(Var, BlockStmt, Context)) + const CheckContext &Ctx, const VarDecl *ObjectArg) { + bool IsConstQualified = Ctx.Var.getType().isConstQualified(); + if (!IsConstQualified && !Ctx.IsVarOnlyUsedAsConst) return; if (ObjectArg != nullptr && - !isInitializingVariableImmutable(*ObjectArg, BlockStmt, Context, + !isInitializingVariableImmutable(*ObjectArg, Ctx.BlockStmt, Ctx.ASTCtx, ExcludedContainerTypes)) return; - if (isVariableUnused(Var, BlockStmt, Context)) { - auto Diagnostic = - diag(Var.getLocation(), - "the %select{|const qualified }0variable %1 is copy-constructed " - "from a const reference but is never used; consider " - "removing the statement") - << IsConstQualified << &Var; - if (IssueFix) - recordRemoval(Stmt, Context, Diagnostic); - } else { - auto Diagnostic = - diag(Var.getLocation(), - "the %select{|const qualified }0variable %1 is copy-constructed " - "from a const reference%select{ but is only used as const " - "reference|}0; consider making it a const reference") - << IsConstQualified << &Var; - if (IssueFix) - recordFixes(Var, Context, Diagnostic); - } + diagnoseCopyFromMethodReturn(Ctx); } void UnnecessaryCopyInitialization::handleCopyFromLocalVar( - const VarDecl &NewVar, const VarDecl &OldVar, const Stmt &BlockStmt, - const DeclStmt &Stmt, bool IssueFix, ASTContext &Context) { - if (!isOnlyUsedAsConst(NewVar, BlockStmt, Context) || - !isInitializingVariableImmutable(OldVar, BlockStmt, Context, + const CheckContext &Ctx, const VarDecl &OldVar) { + if (!Ctx.IsVarOnlyUsedAsConst || + !isInitializingVariableImmutable(OldVar, Ctx.BlockStmt, Ctx.ASTCtx, ExcludedContainerTypes)) return; + diagnoseCopyFromLocalVar(Ctx, OldVar); +} - if (isVariableUnused(NewVar, BlockStmt, Context)) { - auto Diagnostic = diag(NewVar.getLocation(), - "local copy %0 of the variable %1 is never modified " - "and never used; " - "consider removing the statement") - << &NewVar << &OldVar; - if (IssueFix) - recordRemoval(Stmt, Context, Diagnostic); - } else { - auto Diagnostic = - diag(NewVar.getLocation(), - "local copy %0 of the variable %1 is never modified; " - "consider avoiding the copy") - << &NewVar << &OldVar; - if (IssueFix) - recordFixes(NewVar, Context, Diagnostic); +void UnnecessaryCopyInitialization::diagnoseCopyFromMethodReturn( + const CheckContext &Ctx) { + auto Diagnostic = + diag(Ctx.Var.getLocation(), + "the %select{|const qualified }0variable %1 is " + "copy-constructed " + "from a const reference%select{%select{ but is only used as const " + "reference|}0| but is never used}2; consider " + "%select{making it a const reference|removing the statement}2") + << Ctx.Var.getType().isConstQualified() << &Ctx.Var << Ctx.IsVarUnused; + maybeIssueFixes(Ctx, Diagnostic); +} + +void UnnecessaryCopyInitialization::diagnoseCopyFromLocalVar( + const CheckContext &Ctx, const VarDecl &OldVar) { + auto Diagnostic = + diag(Ctx.Var.getLocation(), + "local copy %1 of the variable %0 is never modified%select{" + "| and never used}2; consider %select{avoiding the copy|removing " + "the statement}2") + << &OldVar << &Ctx.Var << Ctx.IsVarUnused; + maybeIssueFixes(Ctx, Diagnostic); +} + +void UnnecessaryCopyInitialization::maybeIssueFixes( + const CheckContext &Ctx, DiagnosticBuilder &Diagnostic) { + if (Ctx.IssueFix) { + if (Ctx.IsVarUnused) + recordRemoval(Ctx.VarDeclStmt, Ctx.ASTCtx, Diagnostic); + else + recordFixes(Ctx.Var, Ctx.ASTCtx, Diagnostic); } } diff --git a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h index ea009ba9979d..ab0f1ecf6106 100644 --- a/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h +++ b/clang-tools-extra/clang-tidy/performance/UnnecessaryCopyInitialization.h @@ -32,14 +32,32 @@ public: void check(const ast_matchers::MatchFinder::MatchResult &Result) override; void storeOptions(ClangTidyOptions::OptionMap &Opts) override; +protected: + // A helper to manipulate the state common to + // `CopyFromMethodReturn` and `CopyFromLocalVar`. + struct CheckContext { + const VarDecl &Var; + const Stmt &BlockStmt; + const DeclStmt &VarDeclStmt; + clang::ASTContext &ASTCtx; + const bool IssueFix; + const bool IsVarUnused; + const bool IsVarOnlyUsedAsConst; + }; + + // Create diagnostics. These are virtual so that derived classes can change + // behaviour. + virtual void diagnoseCopyFromMethodReturn(const CheckContext &Ctx); + virtual void diagnoseCopyFromLocalVar(const CheckContext &Ctx, + const VarDecl &OldVar); + private: - void handleCopyFromMethodReturn(const VarDecl &Var, const Stmt &BlockStmt, - const DeclStmt &Stmt, bool IssueFix, - const VarDecl *ObjectArg, - ASTContext &Context); - void handleCopyFromLocalVar(const VarDecl &NewVar, const VarDecl &OldVar, - const Stmt &BlockStmt, const DeclStmt &Stmt, - bool IssueFix, ASTContext &Context); + void handleCopyFromMethodReturn(const CheckContext &Ctx, + const VarDecl *ObjectArg); + void handleCopyFromLocalVar(const CheckContext &Ctx, const VarDecl &OldVar); + + void maybeIssueFixes(const CheckContext &Ctx, DiagnosticBuilder &Diagnostic); + const std::vector AllowedTypes; const std::vector ExcludedContainerTypes; }; diff --git a/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp b/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp index 831614148c7c..759cdd44fd65 100644 --- a/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp +++ b/clang-tools-extra/clang-tidy/readability/FunctionCognitiveComplexityCheck.cpp @@ -20,6 +20,7 @@ #include "clang/Basic/DiagnosticIDs.h" #include "clang/Basic/LLVM.h" #include "clang/Basic/SourceLocation.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Casting.h" #include "llvm/Support/ErrorHandling.h" @@ -167,15 +168,13 @@ static const std::array Msgs = {{ // Criteria is a bitset, thus a few helpers are needed. CognitiveComplexity::Criteria operator|(CognitiveComplexity::Criteria LHS, CognitiveComplexity::Criteria RHS) { - return static_cast( - static_cast>(LHS) | - static_cast>(RHS)); + return static_cast(llvm::to_underlying(LHS) | + llvm::to_underlying(RHS)); } CognitiveComplexity::Criteria operator&(CognitiveComplexity::Criteria LHS, CognitiveComplexity::Criteria RHS) { - return static_cast( - static_cast>(LHS) & - static_cast>(RHS)); + return static_cast(llvm::to_underlying(LHS) & + llvm::to_underlying(RHS)); } CognitiveComplexity::Criteria &operator|=(CognitiveComplexity::Criteria &LHS, CognitiveComplexity::Criteria RHS) { diff --git a/clang-tools-extra/clangd/IncludeCleaner.cpp b/clang-tools-extra/clangd/IncludeCleaner.cpp index b0a3c290bad6..dda7c9f581f6 100644 --- a/clang-tools-extra/clangd/IncludeCleaner.cpp +++ b/clang-tools-extra/clangd/IncludeCleaner.cpp @@ -397,10 +397,10 @@ IncludeCleanerFindings computeIncludeCleanerFindings(ParsedAST &AST) { std::vector MissingIncludes; llvm::DenseSet Used; trace::Span Tracer("include_cleaner::walkUsed"); - const DirectoryEntry *ResourceDir = AST.getPreprocessor() - .getHeaderSearchInfo() - .getModuleMap() - .getBuiltinDir(); + OptionalDirectoryEntryRef ResourceDir = AST.getPreprocessor() + .getHeaderSearchInfo() + .getModuleMap() + .getBuiltinDir(); include_cleaner::walkUsed( AST.getLocalTopLevelDecls(), /*MacroRefs=*/Macros, AST.getPragmaIncludes().get(), AST.getPreprocessor(), diff --git a/clang-tools-extra/clangd/SemanticHighlighting.cpp b/clang-tools-extra/clangd/SemanticHighlighting.cpp index 49e479abf456..37939d36425a 100644 --- a/clang-tools-extra/clangd/SemanticHighlighting.cpp +++ b/clang-tools-extra/clangd/SemanticHighlighting.cpp @@ -418,7 +418,8 @@ class HighlightingsBuilder { public: HighlightingsBuilder(const ParsedAST &AST, const HighlightingFilter &Filter) : TB(AST.getTokens()), SourceMgr(AST.getSourceManager()), - LangOpts(AST.getLangOpts()), Filter(Filter) {} + LangOpts(AST.getLangOpts()), Filter(Filter), + Resolver(AST.getHeuristicResolver()) {} HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) { auto Range = getRangeForSourceLocation(Loc); @@ -589,7 +590,7 @@ private: HighlightingFilter Filter; std::vector Tokens; std::map> ExtraModifiers; - const HeuristicResolver *Resolver = nullptr; + const HeuristicResolver *Resolver; // returned from addToken(InvalidLoc) HighlightingToken InvalidHighlightingToken; }; diff --git a/clang-tools-extra/docs/ReleaseNotes.rst b/clang-tools-extra/docs/ReleaseNotes.rst index 9262f9bbfe62..6d91748e4cef 100644 --- a/clang-tools-extra/docs/ReleaseNotes.rst +++ b/clang-tools-extra/docs/ReleaseNotes.rst @@ -174,6 +174,12 @@ New checks Flags coroutines that suspend while a lock guard is in scope at the suspension point. +- New :doc:`hicpp-ignored-remove-result + ` check. + + Ensure that the result of ``std::remove``, ``std::remove_if`` and + ``std::unique`` are not ignored according to rule 17.5.1. + - New :doc:`misc-coroutine-hostile-raii ` check. diff --git a/clang-tools-extra/docs/clang-tidy/checks/hicpp/ignored-remove-result.rst b/clang-tools-extra/docs/clang-tidy/checks/hicpp/ignored-remove-result.rst new file mode 100644 index 000000000000..6ca704ae3e66 --- /dev/null +++ b/clang-tools-extra/docs/clang-tidy/checks/hicpp/ignored-remove-result.rst @@ -0,0 +1,24 @@ +.. title:: clang-tidy - hicpp-ignored-remove-result + +hicpp-ignored-remove-result +=========================== + +Ensure that the result of ``std::remove``, ``std::remove_if`` and ``std::unique`` +are not ignored according to +`rule 17.5.1 `_. + +The mutating algorithms ``std::remove``, ``std::remove_if`` and both overloads +of ``std::unique`` operate by swapping or moving elements of the range they are +operating over. On completion, they return an iterator to the last valid +element. In the majority of cases the correct behavior is to use this result as +the first operand in a call to ``std::erase``. + +This check is a subset of :doc:`bugprone-unused-return-value <../bugprone/unused-return-value>` +and depending on used options it can be superfluous to enable both checks. + +Options +------- + +.. option:: AllowCastToVoid + + Controls whether casting return values to ``void`` is permitted. Default: `true`. diff --git a/clang-tools-extra/docs/clang-tidy/checks/list.rst b/clang-tools-extra/docs/clang-tidy/checks/list.rst index e6c02fe48fbf..31f0e090db1d 100644 --- a/clang-tools-extra/docs/clang-tidy/checks/list.rst +++ b/clang-tools-extra/docs/clang-tidy/checks/list.rst @@ -226,6 +226,7 @@ Clang-Tidy Checks :doc:`google-runtime-operator `, :doc:`google-upgrade-googletest-case `, "Yes" :doc:`hicpp-exception-baseclass `, + :doc:`hicpp-ignored-remove-result `, :doc:`hicpp-multiway-paths-covered `, :doc:`hicpp-no-assembler `, :doc:`hicpp-signed-bitwise `, diff --git a/clang-tools-extra/include-cleaner/lib/Analysis.cpp b/clang-tools-extra/include-cleaner/lib/Analysis.cpp index 09365c36f9f2..450c4c796c14 100644 --- a/clang-tools-extra/include-cleaner/lib/Analysis.cpp +++ b/clang-tools-extra/include-cleaner/lib/Analysis.cpp @@ -87,7 +87,7 @@ analyze(llvm::ArrayRef ASTRoots, llvm::StringSet<> Missing; if (!HeaderFilter) HeaderFilter = [](llvm::StringRef) { return false; }; - const DirectoryEntry *ResourceDir = + OptionalDirectoryEntryRef ResourceDir = PP.getHeaderSearchInfo().getModuleMap().getBuiltinDir(); walkUsed(ASTRoots, MacroRefs, PI, PP, [&](const SymbolReference &Ref, llvm::ArrayRef
Providers) { @@ -95,7 +95,7 @@ analyze(llvm::ArrayRef ASTRoots, for (const Header &H : Providers) { if (H.kind() == Header::Physical && (H.physical() == MainFile || - H.physical().getDir() == ResourceDir)) { + (ResourceDir && H.physical().getDir() == *ResourceDir))) { Satisfied = true; } for (const Include *I : Inc.match(H)) { @@ -114,7 +114,7 @@ analyze(llvm::ArrayRef ASTRoots, for (const Include &I : Inc.all()) { if (Used.contains(&I) || !I.Resolved || HeaderFilter(I.Resolved->getFileEntry().tryGetRealPathName()) || - I.Resolved->getFileEntry().getDir() == ResourceDir) + (ResourceDir && I.Resolved->getFileEntry().getDir() == *ResourceDir)) continue; if (PI) { if (PI->shouldKeep(*I.Resolved)) diff --git a/clang-tools-extra/modularize/ModuleAssistant.cpp b/clang-tools-extra/modularize/ModuleAssistant.cpp index 0d4c09987eb1..5c11ffdb8589 100644 --- a/clang-tools-extra/modularize/ModuleAssistant.cpp +++ b/clang-tools-extra/modularize/ModuleAssistant.cpp @@ -175,7 +175,7 @@ static bool addModuleDescription(Module *RootModule, llvm::SmallString<256> NativePath, NativePrefix; llvm::sys::path::native(HeaderFilePath, NativePath); llvm::sys::path::native(HeaderPrefix, NativePrefix); - if (NativePath.startswith(NativePrefix)) + if (NativePath.starts_with(NativePrefix)) FilePath = std::string(NativePath.substr(NativePrefix.size() + 1)); else FilePath = std::string(HeaderFilePath); diff --git a/clang-tools-extra/pseudo/include/clang-pseudo/Token.h b/clang-tools-extra/pseudo/include/clang-pseudo/Token.h index 22b72c71cbba..859fd7d2b3df 100644 --- a/clang-tools-extra/pseudo/include/clang-pseudo/Token.h +++ b/clang-tools-extra/pseudo/include/clang-pseudo/Token.h @@ -32,6 +32,7 @@ #include "clang/Basic/LangStandard.h" #include "clang/Basic/TokenKinds.h" #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/Support/raw_ostream.h" #include #include @@ -71,10 +72,10 @@ struct Token { Index OriginalIndex = Invalid; // Helpers to get/set Flags based on `enum class`. template bool flag(T Mask) const { - return Flags & uint8_t{static_cast>(Mask)}; + return Flags & uint8_t{llvm::to_underlying(Mask)}; } template void setFlag(T Mask) { - Flags |= uint8_t{static_cast>(Mask)}; + Flags |= uint8_t{llvm::to_underlying(Mask)}; } /// Returns the next token in the stream. this may not be a sentinel. diff --git a/clang-tools-extra/test/clang-tidy/checkers/hicpp/ignored-remove-result.cpp b/clang-tools-extra/test/clang-tidy/checkers/hicpp/ignored-remove-result.cpp new file mode 100644 index 000000000000..b068f0859098 --- /dev/null +++ b/clang-tools-extra/test/clang-tidy/checkers/hicpp/ignored-remove-result.cpp @@ -0,0 +1,66 @@ +// RUN: %check_clang_tidy %s hicpp-ignored-remove-result %t +// RUN: %check_clang_tidy -check-suffixes=NOCAST %s hicpp-ignored-remove-result %t -- -config='{CheckOptions: {hicpp-ignored-remove-result.AllowCastToVoid: false}}' + +namespace std { + +template +ForwardIt remove(ForwardIt, ForwardIt, const T &); + +template +ForwardIt remove_if(ForwardIt, ForwardIt, UnaryPredicate); + +template +ForwardIt unique(ForwardIt, ForwardIt); + +template +InputIt find(InputIt, InputIt, const T&); + +class error_code { +}; + +} // namespace std + +std::error_code errorFunc() { + return std::error_code(); +} + +void warning() { + std::remove(nullptr, nullptr, 1); + // CHECK-MESSAGES: [[@LINE-1]]:3: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors + // CHECK-MESSAGES: [[@LINE-2]]:3: note: cast the expression to void to silence this warning + // CHECK-MESSAGES-NOCAST: [[@LINE-3]]:3: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors + + std::remove_if(nullptr, nullptr, nullptr); + // CHECK-MESSAGES: [[@LINE-1]]:3: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors + // CHECK-MESSAGES: [[@LINE-2]]:3: note: cast the expression to void to silence this warning + // CHECK-MESSAGES-NOCAST: [[@LINE-3]]:3: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors + + std::unique(nullptr, nullptr); + // CHECK-MESSAGES: [[@LINE-1]]:3: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors + // CHECK-MESSAGES: [[@LINE-2]]:3: note: cast the expression to void to silence this warning + // CHECK-MESSAGES-NOCAST: [[@LINE-3]]:3: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors +} + +void optionalWarning() { + // No warning unless AllowCastToVoid=false + (void)std::remove(nullptr, nullptr, 1); + // CHECK-MESSAGES-NOCAST: [[@LINE-1]]:9: warning: the value returned by this function should not be disregarded; neglecting it may lead to errors +} + +void noWarning() { + + auto RemoveRetval = std::remove(nullptr, nullptr, 1); + + auto RemoveIfRetval = std::remove_if(nullptr, nullptr, nullptr); + + auto UniqueRetval = std::unique(nullptr, nullptr); + + // Verify that other checks in the baseclass are not used. + // - no warning on std::find since the checker overrides + // bugprone-unused-return-value's checked functions. + std::find(nullptr, nullptr, 1); + // - no warning on return types since the checker disable + // bugprone-unused-return-value's checked return types. + errorFunc(); + (void) errorFunc(); +} diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst index 286405850900..d34e867f5e61 100644 --- a/clang/docs/LanguageExtensions.rst +++ b/clang/docs/LanguageExtensions.rst @@ -1483,6 +1483,7 @@ Conditional ``explicit`` __cpp_conditional_explicit C++20 ``using enum`` __cpp_using_enum C++20 C++03 ``if consteval`` __cpp_if_consteval C++23 C++20 ``static operator()`` __cpp_static_call_operator C++23 C++03 +Attributes on Lambda-Expressions C++23 C++11 -------------------------------------- -------------------------------- ------------- ------------- Designated initializers (N494) C99 C89 Array & element qualification (N2607) C23 C89 @@ -3866,6 +3867,30 @@ builtin function, and are named with a ``__opencl_`` prefix. The macros and ``__OPENCL_MEMORY_SCOPE_SUB_GROUP`` are provided, with values corresponding to the enumerators of OpenCL's ``memory_scope`` enumeration.) +__scoped_atomic builtins +------------------------ + +Clang provides a set of atomics taking a memory scope argument. These atomics +are identical to the standard GNU / GCC atomic builtins but taking an extra +memory scope argument. These are designed to be a generic alternative to the +``__opencl_atomic_*`` builtin functions for targets that support atomic memory +scopes. + +Atomic memory scopes are designed to assist optimizations for systems with +several levels of memory hierarchy like GPUs. The following memory scopes are +currently supported: + +* ``__MEMORY_SCOPE_SYSTEM`` +* ``__MEMORY_SCOPE_DEVICE`` +* ``__MEMORY_SCOPE_WRKGRP`` +* ``__MEMORY_SCOPE_WVFRNT`` +* ``__MEMORY_SCOPE_SINGLE`` + +This controls whether or not the atomic operation is ordered with respect to the +whole system, the current device, an OpenCL workgroup, wavefront, or just a +single thread. If these are used on a target that does not support atomic +scopes, then they will behave exactly as the standard GNU atomic builtins. + Low-level ARM exclusive memory builtins --------------------------------------- diff --git a/clang/docs/ReleaseNotes.rst b/clang/docs/ReleaseNotes.rst index 89ea2f0930ce..783dc7333af7 100644 --- a/clang/docs/ReleaseNotes.rst +++ b/clang/docs/ReleaseNotes.rst @@ -156,6 +156,9 @@ C++23 Feature Support support for this feature is still experimental, the feature test macro ``__cpp_explicit_this_parameter`` was not set in this version. +- Added a separate warning to warn the use of attributes on lambdas as a C++23 extension + in previous language versions: ``-Wc++23-lambda-attributes``. + C++2c Feature Support ^^^^^^^^^^^^^^^^^^^^^ @@ -201,6 +204,9 @@ C Language Changes number of elements in the flexible array member. This information can improve the results of the array bound sanitizer and the ``__builtin_dynamic_object_size`` builtin. +- Enums will now be represented in TBAA metadata using their actual underlying + integer type. Previously they were treated as chars, which meant they could + alias with all other types. C23 Feature Support ^^^^^^^^^^^^^^^^^^^ @@ -232,6 +238,8 @@ Non-comprehensive list of changes in this release except that it returns the size of a type ignoring tail padding. * ``__builtin_classify_type()`` now classifies ``_BitInt`` values as the return value ``18`` and vector types as return value ``19``, to match GCC 14's behavior. +* The default value of `_MSC_VER` was raised from 1920 to 1933. +* Since MSVC 19.33 added undocumented attribute ``[[msvc::constexpr]]``, this release adds the attribute as well. * Added ``#pragma clang fp reciprocal``. @@ -256,6 +264,16 @@ New Compiler Flags * ``-fopenacc`` was added as a part of the effort to support OpenACC in clang. +* ``-fcx-limited-range`` enables the naive mathematical formulas for complex + division and multiplication with no NaN checking of results. The default is + ``-fno-cx-limited-range``, but this option is enabled by ``-ffast-math``. + +* ``-fcx-fortran-rules`` enables the naive mathematical formulas for complex + multiplication and enables application of Smith's algorithm for complex + division. See SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 + (1962). The default is ``-fno-cx-fortran-rules``. + + Deprecated Compiler Flags ------------------------- @@ -353,6 +371,8 @@ Improvements to Clang's diagnostics of a base class is not called in the constructor of its derived class. - Clang no longer emits ``-Wmissing-variable-declarations`` for variables declared with the ``register`` storage class. +- Clang's ``-Wswitch-default`` flag now diagnoses whenever a ``switch`` statement + does not have a ``default`` label. - Clang's ``-Wtautological-negation-compare`` flag now diagnoses logical tautologies like ``x && !x`` and ``!x || x`` in expressions. This also makes ``-Winfinite-recursion`` diagnose more cases. @@ -503,6 +523,7 @@ Improvements to Clang's diagnostics 48 | static_assert(1 << 4 == 15); | ~~~~~~~^~~~~ +- Clang now diagnoses definitions of friend function specializations, e.g. ``friend void f<>(int) {}``. Improvements to Clang's time-trace ---------------------------------- @@ -651,6 +672,9 @@ Bug Fixes in This Version - Fixed false positive error emitted by clang when performing qualified name lookup and the current class instantiation has dependent bases. Fixes (`#13826 `_) +- Fix a ``clang-17`` regression where a templated friend with constraints is not + properly applied when its parameters reference an enclosing non-template class. + Fixes (`#71595 `_) - Fix the name of the ifunc symbol emitted for multiversion functions declared with the ``target_clones`` attribute. This addresses a linker error that would otherwise occur when these functions are referenced from other TUs. @@ -658,6 +682,12 @@ Bug Fixes in This Version Fixes (`#64467 `_) - Clang's ``-Wchar-subscripts`` no longer warns on chars whose values are known non-negative constants. Fixes (`#18763 `_) +- Fix crash due to incorrectly allowing conversion functions in copy elision. + Fixes (`#39319 `_) and + (`#60182 `_) and + (`#62157 `_) and + (`#64885 `_) and + (`#65568 `_) Bug Fixes to Compiler Builtins ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -886,11 +916,17 @@ Arm and AArch64 Support - New AArch64 asm constraints have been added for r8-r11(Uci) and r12-r15(Ucj). - Support has been added for the following processors (-mcpu identifiers in parenthesis): +- Support has been added for the following processors (-mcpu identifiers in parenthesis): - * Arm Cortex-A520 (cortex-a520). - * Arm Cortex-A720 (cortex-a720). - * Arm Cortex-X4 (cortex-x4). + For Arm: + + * Cortex-M52 (cortex-m52). + + For AArch64: + + * Cortex-A520 (cortex-a520). + * Cortex-A720 (cortex-a720). + * Cortex-X4 (cortex-x4). Android Support ^^^^^^^^^^^^^^^ @@ -937,6 +973,9 @@ CUDA/HIP Language Changes CUDA Support ^^^^^^^^^^^^ +- Clang now supports CUDA SDK up to 12.3 +- Added support for sm_90a + AIX Support ^^^^^^^^^^^ @@ -976,6 +1015,9 @@ Floating Point Support in Clang ``__builtin_exp10f128`` builtins. - Add ``__builtin_iszero``, ``__builtin_issignaling`` and ``__builtin_issubnormal``. +- Add support for C99's ``#pragma STDC CX_LIMITED_RANGE`` feature. This + enables the naive mathematical formulas for complex multiplication and + division, which are faster but do not correctly handle overflow and infinities. AST Matchers ------------ diff --git a/clang/docs/UsersManual.rst b/clang/docs/UsersManual.rst index 9d64195ee338..7c30570437e8 100644 --- a/clang/docs/UsersManual.rst +++ b/clang/docs/UsersManual.rst @@ -1468,6 +1468,7 @@ floating point semantic models: precise (the default), strict, and fast. With the exception of ``-ffp-contract=fast``, using any of the options below to disable any of the individual optimizations in ``-ffast-math`` will cause ``__FAST_MATH__`` to no longer be set. + ``-ffast-math`` enables ``-fcx-limited-range``. This option implies: @@ -1834,6 +1835,20 @@ floating point semantic models: precise (the default), strict, and fast. * ``16`` - Forces ``_Float16`` operations to be emitted without using excess precision arithmetic. +.. option:: -fcx-limited-range: + + This option enables the naive mathematical formulas for complex division and + multiplication with no NaN checking of results. The default is + ``-fno-cx-limited-range``, but this option is enabled by the ``-ffast-math`` + option. + +.. option:: -fcx-fortran-rules: + + This option enables the naive mathematical formulas for complex + multiplication and enables application of Smith's algorithm for complex + division. See SMITH, R. L. Algorithm 116: Complex division. Commun. + ACM 5, 8 (1962). The default is ``-fno-cx-fortran-rules``. + .. _floating-point-environment: Accessing the floating point environment @@ -3359,8 +3374,8 @@ default for Windows targets. For compatibility with existing code that compiles with MSVC, clang defines the ``_MSC_VER`` and ``_MSC_FULL_VER`` macros. When on Windows, these default to -either the same value as the currently installed version of cl.exe, or ``1920`` -and ``192000000`` (respectively). The ``-fms-compatibility-version=`` flag +either the same value as the currently installed version of cl.exe, or ``1933`` +and ``193300000`` (respectively). The ``-fms-compatibility-version=`` flag overrides these values. It accepts a dotted version tuple, such as 19.00.23506. Changing the MSVC compatibility version makes clang behave more like that version of MSVC. For example, ``-fms-compatibility-version=19`` will enable diff --git a/clang/include/clang/AST/Expr.h b/clang/include/clang/AST/Expr.h index a9c4c67a60e8..a41f2d66b37b 100644 --- a/clang/include/clang/AST/Expr.h +++ b/clang/include/clang/AST/Expr.h @@ -6498,7 +6498,7 @@ public: return cast(SubExprs[ORDER_FAIL]); } Expr *getVal2() const { - if (Op == AO__atomic_exchange) + if (Op == AO__atomic_exchange || Op == AO__scoped_atomic_exchange) return cast(SubExprs[ORDER_FAIL]); assert(NumSubExprs > VAL2); return cast(SubExprs[VAL2]); @@ -6539,7 +6539,9 @@ public: getOp() == AO__opencl_atomic_compare_exchange_weak || getOp() == AO__hip_atomic_compare_exchange_weak || getOp() == AO__atomic_compare_exchange || - getOp() == AO__atomic_compare_exchange_n; + getOp() == AO__atomic_compare_exchange_n || + getOp() == AO__scoped_atomic_compare_exchange || + getOp() == AO__scoped_atomic_compare_exchange_n; } bool isOpenCL() const { @@ -6569,13 +6571,13 @@ public: /// \return empty atomic scope model if the atomic op code does not have /// scope operand. static std::unique_ptr getScopeModel(AtomicOp Op) { - auto Kind = - (Op >= AO__opencl_atomic_load && Op <= AO__opencl_atomic_fetch_max) - ? AtomicScopeModelKind::OpenCL - : (Op >= AO__hip_atomic_load && Op <= AO__hip_atomic_fetch_max) - ? AtomicScopeModelKind::HIP - : AtomicScopeModelKind::None; - return AtomicScopeModel::create(Kind); + if (Op >= AO__opencl_atomic_load && Op <= AO__opencl_atomic_fetch_max) + return AtomicScopeModel::create(AtomicScopeModelKind::OpenCL); + else if (Op >= AO__hip_atomic_load && Op <= AO__hip_atomic_fetch_max) + return AtomicScopeModel::create(AtomicScopeModelKind::HIP); + else if (Op >= AO__scoped_atomic_load && Op <= AO__scoped_atomic_fetch_max) + return AtomicScopeModel::create(AtomicScopeModelKind::Generic); + return AtomicScopeModel::create(AtomicScopeModelKind::None); } /// Get atomic scope model. diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h index 6c147eb8f640..b3ae66e6e769 100644 --- a/clang/include/clang/AST/Type.h +++ b/clang/include/clang/AST/Type.h @@ -36,6 +36,7 @@ #include "llvm/ADT/FoldingSet.h" #include "llvm/ADT/PointerIntPair.h" #include "llvm/ADT/PointerUnion.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/iterator_range.h" @@ -2384,8 +2385,6 @@ public: bool isRVVType(unsigned ElementCount) const; - bool isRVVType() const; - bool isRVVType(unsigned Bitwidth, bool IsFloat, bool IsBFloat = false) const; /// Return the implicit lifetime for this type, which must not be dependent. @@ -7284,14 +7283,6 @@ inline bool Type::isOpenCLSpecificType() const { isQueueT() || isReserveIDT() || isPipeType() || isOCLExtOpaqueType(); } -inline bool Type::isRVVType() const { -#define RVV_TYPE(Name, Id, SingletonId) \ - isSpecificBuiltinType(BuiltinType::Id) || - return -#include "clang/Basic/RISCVVTypes.def" - false; // end of boolean or operation. -} - inline bool Type::isRVVType(unsigned ElementCount) const { bool Ret = false; #define RVV_VECTOR_TYPE(Name, Id, SingletonId, NumEls, ElBits, NF, IsSigned, \ @@ -7524,7 +7515,7 @@ inline const Type *Type::getPointeeOrArrayElementType() const { /// spaces into a diagnostic with <<. inline const StreamingDiagnostic &operator<<(const StreamingDiagnostic &PD, LangAS AS) { - PD.AddTaggedVal(static_cast>(AS), + PD.AddTaggedVal(llvm::to_underlying(AS), DiagnosticsEngine::ArgumentKind::ak_addrspace); return PD; } diff --git a/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def b/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def index ff687a0d178b..757ee452ced7 100644 --- a/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def +++ b/clang/include/clang/Analysis/Analyses/UnsafeBufferUsageGadgets.def @@ -36,6 +36,7 @@ FIXABLE_GADGET(PointerDereference) FIXABLE_GADGET(UPCAddressofArraySubscript) // '&DRE[any]' in an Unspecified Pointer Context FIXABLE_GADGET(UPCStandalonePointer) FIXABLE_GADGET(UPCPreIncrement) // '++Ptr' in an Unspecified Pointer Context +FIXABLE_GADGET(UUCAddAssign) // 'Ptr += n' in an Unspecified Untyped Context FIXABLE_GADGET(PointerAssignment) FIXABLE_GADGET(PointerInit) diff --git a/clang/include/clang/Basic/Attr.td b/clang/include/clang/Basic/Attr.td index 121ed203829c..0d94ea2851c9 100644 --- a/clang/include/clang/Basic/Attr.td +++ b/clang/include/clang/Basic/Attr.td @@ -3646,6 +3646,14 @@ def : MutualExclusions<[Owner, Pointer]>; // Microsoft-related attributes +def MSConstexpr : InheritableAttr { + let LangOpts = [MicrosoftExt]; + let Spellings = [CXX11<"msvc", "constexpr">]; + let Subjects = SubjectList<[Function, ReturnStmt], ErrorDiag, + "functions and return statements">; + let Documentation = [MSConstexprDocs]; +} + def MSNoVTable : InheritableAttr, TargetSpecificAttr { let Spellings = [Declspec<"novtable">]; let Subjects = SubjectList<[CXXRecord]>; @@ -4250,7 +4258,8 @@ def HLSLResource : InheritableAttr { "StructuredBuffer", "CBuffer", "Sampler", "TBuffer", "RTAccelerationStructure", "FeedbackTexture2D", "FeedbackTexture2DArray"], - /*opt=*/0, /*fake=*/0, /*isExternalType=*/1> + /*opt=*/0, /*fake=*/0, /*isExternalType=*/1>, + DefaultBoolArgument<"isROV", /*default=*/0> ]; let Documentation = [InternalOnly]; } diff --git a/clang/include/clang/Basic/AttrDocs.td b/clang/include/clang/Basic/AttrDocs.td index bbe4de94cbab..1a98196834ce 100644 --- a/clang/include/clang/Basic/AttrDocs.td +++ b/clang/include/clang/Basic/AttrDocs.td @@ -2659,8 +2659,9 @@ An error will be given if: - Specified values violate subtarget specifications; - Specified values are not compatible with values provided through other attributes; - - The AMDGPU target backend is unable to create machine code that can meet the - request. + +The AMDGPU target backend will emit a warning whenever it is unable to +create machine code that meets the request. }]; } @@ -3656,6 +3657,21 @@ an error: }]; } +def MSConstexprDocs : Documentation { + let Category = DocCatStmt; + let Content = [{ +The ``[[msvc::constexpr]]`` attribute can be applied only to a function +definition or a ``return`` statement. It does not impact function declarations. +A ``[[msvc::constexpr]]`` function cannot be ``constexpr`` or ``consteval``. +A ``[[msvc::constexpr]]`` function is treated as if it were a ``constexpr`` function +when it is evaluated in a constant context of ``[[msvc::constexpr]] return`` statement. +Otherwise, it is treated as a regular function. + +Semantics of this attribute are enabled only under MSVC compatibility +(``-fms-compatibility-version``) 19.33 and later. + }]; +} + def MSNoVTableDocs : Documentation { let Category = DocCatDecl; let Content = [{ diff --git a/clang/include/clang/Basic/Builtins.def b/clang/include/clang/Basic/Builtins.def index ec39e926889b..4dcbaf8a7bea 100644 --- a/clang/include/clang/Basic/Builtins.def +++ b/clang/include/clang/Basic/Builtins.def @@ -904,6 +904,32 @@ BUILTIN(__atomic_signal_fence, "vi", "n") BUILTIN(__atomic_always_lock_free, "bzvCD*", "nE") BUILTIN(__atomic_is_lock_free, "bzvCD*", "nE") +// GNU atomic builtins with atomic scopes. +ATOMIC_BUILTIN(__scoped_atomic_load, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_load_n, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_store, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_store_n, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_exchange, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_exchange_n, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_compare_exchange, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_compare_exchange_n, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_add, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_sub, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_and, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_or, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_xor, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_nand, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_add_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_sub_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_and_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_or_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_xor_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_max_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_min_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_nand_fetch, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_min, "v.", "t") +ATOMIC_BUILTIN(__scoped_atomic_fetch_max, "v.", "t") + // OpenCL 2.0 atomic builtins. ATOMIC_BUILTIN(__opencl_atomic_init, "v.", "t") ATOMIC_BUILTIN(__opencl_atomic_load, "v.", "t") diff --git a/clang/include/clang/Basic/BuiltinsNVPTX.def b/clang/include/clang/Basic/BuiltinsNVPTX.def index d74a7d1e55dd..0f2e8260143b 100644 --- a/clang/include/clang/Basic/BuiltinsNVPTX.def +++ b/clang/include/clang/Basic/BuiltinsNVPTX.def @@ -26,7 +26,9 @@ #pragma push_macro("SM_87") #pragma push_macro("SM_89") #pragma push_macro("SM_90") -#define SM_90 "sm_90" +#pragma push_macro("SM_90a") +#define SM_90a "sm_90a" +#define SM_90 "sm_90|" SM_90a #define SM_89 "sm_89|" SM_90 #define SM_87 "sm_87|" SM_89 #define SM_86 "sm_86|" SM_87 @@ -56,7 +58,11 @@ #pragma push_macro("PTX78") #pragma push_macro("PTX80") #pragma push_macro("PTX81") -#define PTX81 "ptx81" +#pragma push_macro("PTX82") +#pragma push_macro("PTX83") +#define PTX83 "ptx83" +#define PTX82 "ptx82|" PTX83 +#define PTX81 "ptx81|" PTX82 #define PTX80 "ptx80|" PTX81 #define PTX78 "ptx78|" PTX80 #define PTX77 "ptx77|" PTX78 @@ -1055,6 +1061,7 @@ TARGET_BUILTIN(__nvvm_getctarank_shared_cluster, "iv*3", "", AND(SM_90,PTX78)) #pragma pop_macro("SM_87") #pragma pop_macro("SM_89") #pragma pop_macro("SM_90") +#pragma pop_macro("SM_90a") #pragma pop_macro("PTX42") #pragma pop_macro("PTX60") #pragma pop_macro("PTX61") @@ -1072,3 +1079,5 @@ TARGET_BUILTIN(__nvvm_getctarank_shared_cluster, "iv*3", "", AND(SM_90,PTX78)) #pragma pop_macro("PTX78") #pragma pop_macro("PTX80") #pragma pop_macro("PTX81") +#pragma pop_macro("PTX82") +#pragma pop_macro("PTX83") diff --git a/clang/include/clang/Basic/Cuda.h b/clang/include/clang/Basic/Cuda.h index 2d912bdbbd1b..916cb4b7ef34 100644 --- a/clang/include/clang/Basic/Cuda.h +++ b/clang/include/clang/Basic/Cuda.h @@ -39,9 +39,11 @@ enum class CudaVersion { CUDA_118, CUDA_120, CUDA_121, - FULLY_SUPPORTED = CUDA_118, + CUDA_122, + CUDA_123, + FULLY_SUPPORTED = CUDA_123, PARTIALLY_SUPPORTED = - CUDA_121, // Partially supported. Proceed with a warning. + CUDA_123, // Partially supported. Proceed with a warning. NEW = 10000, // Too new. Issue a warning, but allow using it. }; const char *CudaVersionToString(CudaVersion V); @@ -71,6 +73,7 @@ enum class CudaArch { SM_87, SM_89, SM_90, + SM_90a, GFX600, GFX601, GFX602, diff --git a/clang/include/clang/Basic/DiagnosticFrontendKinds.td b/clang/include/clang/Basic/DiagnosticFrontendKinds.td index 715e0c0dc8fa..568000106a84 100644 --- a/clang/include/clang/Basic/DiagnosticFrontendKinds.td +++ b/clang/include/clang/Basic/DiagnosticFrontendKinds.td @@ -80,6 +80,7 @@ def remark_fe_backend_optimization_remark_analysis_aliasing : Remark<"%0; " "the '__restrict__' qualifier with the independent array arguments. " "Erroneous results will occur if these options are incorrectly applied!">, BackendInfo, InGroup; + def warn_fe_backend_optimization_failure : Warning<"%0">, BackendInfo, InGroup, DefaultWarn; def note_fe_backend_invalid_loc : Note<"could " diff --git a/clang/include/clang/Basic/DiagnosticGroups.td b/clang/include/clang/Basic/DiagnosticGroups.td index ff028bbbf742..caee2dc6daad 100644 --- a/clang/include/clang/Basic/DiagnosticGroups.td +++ b/clang/include/clang/Basic/DiagnosticGroups.td @@ -632,7 +632,7 @@ def ShadowAll : DiagGroup<"shadow-all", [Shadow, ShadowFieldInConstructor, def Shorten64To32 : DiagGroup<"shorten-64-to-32">; def : DiagGroup<"sign-promo">; def SignCompare : DiagGroup<"sign-compare">; -def : DiagGroup<"switch-default">; +def SwitchDefault : DiagGroup<"switch-default">; def : DiagGroup<"synth">; def SizeofArrayArgument : DiagGroup<"sizeof-array-argument">; def SizeofArrayDecay : DiagGroup<"sizeof-array-decay">; @@ -1126,6 +1126,8 @@ def FutureAttrs : DiagGroup<"future-attribute-extensions", [CXX14Attrs, CXX17Attrs, CXX20Attrs]>; +def CXX23AttrsOnLambda : DiagGroup<"c++23-lambda-attributes">; + // A warning group for warnings about using C++11 features as extensions in // earlier C++ versions. def CXX11 : DiagGroup<"c++11-extensions", [CXX11ExtraSemi, CXX11InlineNamespace, @@ -1145,7 +1147,7 @@ def CXX20 : DiagGroup<"c++20-extensions", [CXX20Designator, CXX20Attrs]>; // A warning group for warnings about using C++23 features as extensions in // earlier C++ versions. -def CXX23 : DiagGroup<"c++23-extensions">; +def CXX23 : DiagGroup<"c++23-extensions", [CXX23AttrsOnLambda]>; // A warning group for warnings about using C++26 features as extensions in // earlier C++ versions. diff --git a/clang/include/clang/Basic/DiagnosticParseKinds.td b/clang/include/clang/Basic/DiagnosticParseKinds.td index e0e199e0d730..6150fc36430a 100644 --- a/clang/include/clang/Basic/DiagnosticParseKinds.td +++ b/clang/include/clang/Basic/DiagnosticParseKinds.td @@ -1035,7 +1035,7 @@ def err_capture_default_first : Error< "capture default must be first">; def ext_decl_attrs_on_lambda : ExtWarn< "%select{an attribute specifier sequence|%0}1 in this position " - "is a C++23 extension">, InGroup; + "is a C++23 extension">, InGroup; def ext_lambda_missing_parens : ExtWarn< "lambda without a parameter clause is a C++23 extension">, InGroup; diff --git a/clang/include/clang/Basic/DiagnosticSemaKinds.td b/clang/include/clang/Basic/DiagnosticSemaKinds.td index 6dfb2d719520..94e97a891bae 100644 --- a/clang/include/clang/Basic/DiagnosticSemaKinds.td +++ b/clang/include/clang/Basic/DiagnosticSemaKinds.td @@ -1669,6 +1669,8 @@ def err_qualified_friend_def : Error< "friend function definition cannot be qualified with '%0'">; def err_friend_def_in_local_class : Error< "friend function cannot be defined in a local class">; +def err_friend_specialization_def : Error< + "friend function specialization cannot be defined">; def err_friend_not_first_in_declaration : Error< "'friend' must appear first in a non-function declaration">; def err_using_decl_friend : Error< @@ -2884,6 +2886,8 @@ def warn_cxx11_compat_constexpr_body_multiple_return : Warning< InGroup, DefaultIgnore; def note_constexpr_body_previous_return : Note< "previous return statement is here">; +def err_ms_constexpr_cannot_be_applied : Error< + "attribute 'msvc::constexpr' cannot be applied to the %select{constexpr|consteval|virtual}0 function %1">; // C++20 function try blocks in constexpr def ext_constexpr_function_try_block_cxx20 : ExtWarn< @@ -4933,6 +4937,8 @@ def err_template_param_shadow : Error< def ext_template_param_shadow : ExtWarn< err_template_param_shadow.Summary>, InGroup; def note_template_param_here : Note<"template parameter is declared here">; +def note_template_param_external : Note< + "template parameter from hidden source: %0">; def warn_template_export_unsupported : Warning< "exported templates are unsupported">; def err_template_outside_namespace_or_class_scope : Error< @@ -5061,6 +5067,8 @@ def err_template_arg_list_different_arity : Error< "%select{class template|function template|variable template|alias template|" "template template parameter|concept|template}1 %2">; def note_template_decl_here : Note<"template is declared here">; +def note_template_decl_external : Note< + "template declaration from hidden source: %0">; def err_template_arg_must_be_type : Error< "template argument for template type parameter must be a type">; def err_template_arg_must_be_type_suggest : Error< @@ -10060,6 +10068,8 @@ def warn_missing_case : Warning<"%plural{" "3:enumeration values %1, %2, and %3 not handled in switch|" ":%0 enumeration values not handled in switch: %1, %2, %3...}0">, InGroup; +def warn_switch_default : Warning<"'switch' missing 'default' label">, + InGroup, DefaultIgnore; def warn_unannotated_fallthrough : Warning< "unannotated fall-through between switch labels">, @@ -11992,7 +12002,7 @@ def warn_tcb_enforcement_violation : Warning< // RISC-V builtin required extension warning def err_riscv_builtin_requires_extension : Error< - "builtin requires%select{| at least one of the following extensions to be enabled}0: %1">; + "builtin requires%select{| at least one of the following extensions}0: %1">; def err_riscv_builtin_invalid_lmul : Error< "LMUL argument must be in the range [0,3] or [5,7]">; def err_riscv_type_requires_extension : Error< diff --git a/clang/include/clang/Basic/DirectoryEntry.h b/clang/include/clang/Basic/DirectoryEntry.h index 5d083e68facd..906c2e9af23b 100644 --- a/clang/include/clang/Basic/DirectoryEntry.h +++ b/clang/include/clang/Basic/DirectoryEntry.h @@ -245,78 +245,4 @@ template <> struct DenseMapInfo { } // end namespace llvm -namespace clang { - -/// Wrapper around OptionalDirectoryEntryRef that degrades to 'const -/// DirectoryEntry*', facilitating incremental patches to propagate -/// DirectoryEntryRef. -/// -/// This class can be used as return value or field where it's convenient for -/// an OptionalDirectoryEntryRef to degrade to a 'const DirectoryEntry*'. The -/// purpose is to avoid code churn due to dances like the following: -/// \code -/// // Old code. -/// lvalue = rvalue; -/// -/// // Temporary code from an incremental patch. -/// OptionalDirectoryEntryRef MaybeF = rvalue; -/// lvalue = MaybeF ? &MaybeF.getDirectoryEntry() : nullptr; -/// -/// // Final code. -/// lvalue = rvalue; -/// \endcode -/// -/// FIXME: Once DirectoryEntryRef is "everywhere" and DirectoryEntry::LastRef -/// and DirectoryEntry::getName have been deleted, delete this class and -/// replace instances with OptionalDirectoryEntryRef. -class OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr - : public OptionalDirectoryEntryRef { -public: - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr() = default; - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr( - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &&) = default; - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr( - const OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &) = default; - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr & - operator=(OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &&) = default; - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr & - operator=(const OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &) = default; - - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr(std::nullopt_t) {} - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr(DirectoryEntryRef Ref) - : OptionalDirectoryEntryRef(Ref) {} - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr( - OptionalDirectoryEntryRef MaybeRef) - : OptionalDirectoryEntryRef(MaybeRef) {} - - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr & - operator=(std::nullopt_t) { - OptionalDirectoryEntryRef::operator=(std::nullopt); - return *this; - } - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr &operator=(DirectoryEntryRef Ref) { - OptionalDirectoryEntryRef::operator=(Ref); - return *this; - } - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr & - operator=(OptionalDirectoryEntryRef MaybeRef) { - OptionalDirectoryEntryRef::operator=(MaybeRef); - return *this; - } - - /// Degrade to 'const DirectoryEntry *' to allow DirectoryEntry::LastRef and - /// DirectoryEntry::getName have been deleted, delete this class and replace - /// instances with OptionalDirectoryEntryRef - operator const DirectoryEntry *() const { - return has_value() ? &(*this)->getDirEntry() : nullptr; - } -}; - -static_assert(std::is_trivially_copyable< - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr>::value, - "OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr should be " - "trivially copyable"); - -} // end namespace clang - #endif // LLVM_CLANG_BASIC_DIRECTORYENTRY_H diff --git a/clang/include/clang/Basic/FPOptions.def b/clang/include/clang/Basic/FPOptions.def index 5b923a1944e5..79f04c89c9fe 100644 --- a/clang/include/clang/Basic/FPOptions.def +++ b/clang/include/clang/Basic/FPOptions.def @@ -28,4 +28,5 @@ OPTION(FPEvalMethod, LangOptions::FPEvalMethodKind, 2, AllowApproxFunc) OPTION(Float16ExcessPrecision, LangOptions::ExcessPrecisionKind, 2, FPEvalMethod) OPTION(BFloat16ExcessPrecision, LangOptions::ExcessPrecisionKind, 2, Float16ExcessPrecision) OPTION(MathErrno, bool, 1, BFloat16ExcessPrecision) +OPTION(ComplexRange, LangOptions::ComplexRangeKind, 2, MathErrno) #undef OPTION diff --git a/clang/include/clang/Basic/Features.def b/clang/include/clang/Basic/Features.def index adaf2e413f2f..7473e00a7bd8 100644 --- a/clang/include/clang/Basic/Features.def +++ b/clang/include/clang/Basic/Features.def @@ -89,6 +89,8 @@ FEATURE(blocks, LangOpts.Blocks) FEATURE(c_thread_safety_attributes, true) FEATURE(cxx_exceptions, LangOpts.CXXExceptions) FEATURE(cxx_rtti, LangOpts.RTTI &&LangOpts.RTTIData) +EXTENSION(define_target_os_macros, + PP.getPreprocessorOpts().DefineTargetOSMacros) FEATURE(enumerator_attributes, true) FEATURE(nullability, true) FEATURE(nullability_on_arrays, true) @@ -102,6 +104,7 @@ FEATURE(scudo, LangOpts.Sanitize.hasOneOf(SanitizerKind::Scudo)) FEATURE(swiftasynccc, PP.getTargetInfo().checkCallingConvention(CC_SwiftAsync) == clang::TargetInfo::CCCR_OK) +FEATURE(pragma_stdc_cx_limited_range, true) // Objective-C features FEATURE(objc_arr, LangOpts.ObjCAutoRefCount) // FIXME: REMOVE? FEATURE(objc_arc, LangOpts.ObjCAutoRefCount) diff --git a/clang/include/clang/Basic/FileEntry.h b/clang/include/clang/Basic/FileEntry.h index 6351aeae92e2..35efa147950f 100644 --- a/clang/include/clang/Basic/FileEntry.h +++ b/clang/include/clang/Basic/FileEntry.h @@ -279,72 +279,6 @@ template <> struct DenseMapInfo { namespace clang { -/// Wrapper around OptionalFileEntryRef that degrades to 'const FileEntry*', -/// facilitating incremental patches to propagate FileEntryRef. -/// -/// This class can be used as return value or field where it's convenient for -/// an OptionalFileEntryRef to degrade to a 'const FileEntry*'. The purpose -/// is to avoid code churn due to dances like the following: -/// \code -/// // Old code. -/// lvalue = rvalue; -/// -/// // Temporary code from an incremental patch. -/// OptionalFileEntryRef MaybeF = rvalue; -/// lvalue = MaybeF ? &MaybeF.getFileEntry() : nullptr; -/// -/// // Final code. -/// lvalue = rvalue; -/// \endcode -/// -/// FIXME: Once FileEntryRef is "everywhere" and FileEntry::LastRef and -/// FileEntry::getName have been deleted, delete this class and replace -/// instances with OptionalFileEntryRef. -class OptionalFileEntryRefDegradesToFileEntryPtr : public OptionalFileEntryRef { -public: - OptionalFileEntryRefDegradesToFileEntryPtr() = default; - OptionalFileEntryRefDegradesToFileEntryPtr( - OptionalFileEntryRefDegradesToFileEntryPtr &&) = default; - OptionalFileEntryRefDegradesToFileEntryPtr( - const OptionalFileEntryRefDegradesToFileEntryPtr &) = default; - OptionalFileEntryRefDegradesToFileEntryPtr & - operator=(OptionalFileEntryRefDegradesToFileEntryPtr &&) = default; - OptionalFileEntryRefDegradesToFileEntryPtr & - operator=(const OptionalFileEntryRefDegradesToFileEntryPtr &) = default; - - OptionalFileEntryRefDegradesToFileEntryPtr(std::nullopt_t) {} - OptionalFileEntryRefDegradesToFileEntryPtr(FileEntryRef Ref) - : OptionalFileEntryRef(Ref) {} - OptionalFileEntryRefDegradesToFileEntryPtr(OptionalFileEntryRef MaybeRef) - : OptionalFileEntryRef(MaybeRef) {} - - OptionalFileEntryRefDegradesToFileEntryPtr &operator=(std::nullopt_t) { - OptionalFileEntryRef::operator=(std::nullopt); - return *this; - } - OptionalFileEntryRefDegradesToFileEntryPtr &operator=(FileEntryRef Ref) { - OptionalFileEntryRef::operator=(Ref); - return *this; - } - OptionalFileEntryRefDegradesToFileEntryPtr & - operator=(OptionalFileEntryRef MaybeRef) { - OptionalFileEntryRef::operator=(MaybeRef); - return *this; - } - - /// Degrade to 'const FileEntry *' to allow FileEntry::LastRef and - /// FileEntry::getName have been deleted, delete this class and replace - /// instances with OptionalFileEntryRef - operator const FileEntry *() const { - return has_value() ? &(*this)->getFileEntry() : nullptr; - } -}; - -static_assert( - std::is_trivially_copyable< - OptionalFileEntryRefDegradesToFileEntryPtr>::value, - "OptionalFileEntryRefDegradesToFileEntryPtr should be trivially copyable"); - inline bool operator==(const FileEntry *LHS, const OptionalFileEntryRef &RHS) { return LHS == (RHS ? &RHS->getFileEntry() : nullptr); } diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def index c3d5399905a3..152d9f65f86d 100644 --- a/clang/include/clang/Basic/LangOptions.def +++ b/clang/include/clang/Basic/LangOptions.def @@ -220,6 +220,8 @@ BENIGN_LANGOPT(NoSignedZero , 1, 0, "Permit Floating Point optimization wit BENIGN_LANGOPT(AllowRecip , 1, 0, "Permit Floating Point reciprocal") BENIGN_LANGOPT(ApproxFunc , 1, 0, "Permit Floating Point approximation") +ENUM_LANGOPT(ComplexRange, ComplexRangeKind, 2, CX_Full, "Enable use of range reduction for complex arithmetics.") + BENIGN_LANGOPT(ObjCGCBitmapPrint , 1, 0, "printing of GC's bitmap layout for __weak/__strong ivars") BENIGN_LANGOPT(AccessControl , 1, 1, "C++ access control") diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h index 2d167dd2bdf1..9f986fce2d44 100644 --- a/clang/include/clang/Basic/LangOptions.h +++ b/clang/include/clang/Basic/LangOptions.h @@ -152,6 +152,7 @@ public: MSVC2019 = 1920, MSVC2019_5 = 1925, MSVC2019_8 = 1928, + MSVC2022_3 = 1933, }; enum SYCLMajorVersion { @@ -391,6 +392,8 @@ public: IncompleteOnly = 3, }; + enum ComplexRangeKind { CX_Full, CX_Limited, CX_Fortran }; + public: /// The used language standard. LangStandard::Kind LangStd; @@ -740,6 +743,7 @@ public: setAllowFEnvAccess(true); else setAllowFEnvAccess(LangOptions::FPM_Off); + setComplexRange(LO.getComplexRange()); } bool allowFPContractWithinStatement() const { diff --git a/clang/include/clang/Basic/Module.h b/clang/include/clang/Basic/Module.h index d29cc0b45d58..62786e3ac865 100644 --- a/clang/include/clang/Basic/Module.h +++ b/clang/include/clang/Basic/Module.h @@ -156,7 +156,7 @@ public: /// The build directory of this module. This is the directory in /// which the module is notionally built, and relative to which its headers /// are found. - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr Directory; + OptionalDirectoryEntryRef Directory; /// The presumed file name for the module map defining this module. /// Only non-empty when building from preprocessed source. @@ -672,7 +672,7 @@ public: } /// The serialized AST file for this module, if one was created. - OptionalFileEntryRefDegradesToFileEntryPtr getASTFile() const { + OptionalFileEntryRef getASTFile() const { return getTopLevelModule()->ASTFile; } diff --git a/clang/include/clang/Basic/OpenACCKinds.h b/clang/include/clang/Basic/OpenACCKinds.h index 449a75638b43..62c0a4c1a9de 100644 --- a/clang/include/clang/Basic/OpenACCKinds.h +++ b/clang/include/clang/Basic/OpenACCKinds.h @@ -53,7 +53,7 @@ enum class OpenACCDirectiveKind { Shutdown, Set, Update, - // FIXME: wait construct. + Wait, // Procedure Calls in Compute Regions. Routine, diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h index 985ea6354b82..d2ece14da0b1 100644 --- a/clang/include/clang/Basic/SourceManager.h +++ b/clang/include/clang/Basic/SourceManager.h @@ -143,7 +143,7 @@ public: /// /// FIXME: Make non-optional using a virtual file as needed, remove \c /// Filename and use \c OrigEntry.getNameAsRequested() instead. - OptionalFileEntryRefDegradesToFileEntryPtr OrigEntry; + OptionalFileEntryRef OrigEntry; /// References the file which the contents were actually loaded from. /// @@ -1064,8 +1064,8 @@ public: /// Returns the FileEntry record for the provided FileID. const FileEntry *getFileEntryForID(FileID FID) const { - if (auto *Entry = getSLocEntryForFile(FID)) - return Entry->getFile().getContentCache().OrigEntry; + if (auto FE = getFileEntryRefForID(FID)) + return *FE; return nullptr; } @@ -1083,9 +1083,11 @@ public: std::optional getNonBuiltinFilenameForID(FileID FID) const; /// Returns the FileEntry record for the provided SLocEntry. - const FileEntry *getFileEntryForSLocEntry(const SrcMgr::SLocEntry &sloc) const - { - return sloc.getFile().getContentCache().OrigEntry; + const FileEntry * + getFileEntryForSLocEntry(const SrcMgr::SLocEntry &SLocEntry) const { + if (auto FE = SLocEntry.getFile().getContentCache().OrigEntry) + return *FE; + return nullptr; } /// Return a StringRef to the source buffer data for the diff --git a/clang/include/clang/Basic/SyncScope.h b/clang/include/clang/Basic/SyncScope.h index 7919f64c6daf..bc7ec7b5cf77 100644 --- a/clang/include/clang/Basic/SyncScope.h +++ b/clang/include/clang/Basic/SyncScope.h @@ -40,6 +40,11 @@ namespace clang { /// Update getAsString. /// enum class SyncScope { + SystemScope, + DeviceScope, + WorkgroupScope, + WavefrontScope, + SingleScope, HIPSingleThread, HIPWavefront, HIPWorkgroup, @@ -54,6 +59,16 @@ enum class SyncScope { inline llvm::StringRef getAsString(SyncScope S) { switch (S) { + case SyncScope::SystemScope: + return "system_scope"; + case SyncScope::DeviceScope: + return "device_scope"; + case SyncScope::WorkgroupScope: + return "workgroup_scope"; + case SyncScope::WavefrontScope: + return "wavefront_scope"; + case SyncScope::SingleScope: + return "single_scope"; case SyncScope::HIPSingleThread: return "hip_singlethread"; case SyncScope::HIPWavefront: @@ -77,7 +92,7 @@ inline llvm::StringRef getAsString(SyncScope S) { } /// Defines the kind of atomic scope models. -enum class AtomicScopeModelKind { None, OpenCL, HIP }; +enum class AtomicScopeModelKind { None, OpenCL, HIP, Generic }; /// Defines the interface for synch scope model. class AtomicScopeModel { @@ -205,6 +220,56 @@ public: } }; +/// Defines the generic atomic scope model. +class AtomicScopeGenericModel : public AtomicScopeModel { +public: + /// The enum values match predefined built-in macros __ATOMIC_SCOPE_*. + enum ID { + System = 0, + Device = 1, + Workgroup = 2, + Wavefront = 3, + Single = 4, + Last = Single + }; + + AtomicScopeGenericModel() = default; + + SyncScope map(unsigned S) const override { + switch (static_cast(S)) { + case Device: + return SyncScope::DeviceScope; + case System: + return SyncScope::SystemScope; + case Workgroup: + return SyncScope::WorkgroupScope; + case Wavefront: + return SyncScope::WavefrontScope; + case Single: + return SyncScope::SingleScope; + } + llvm_unreachable("Invalid language sync scope value"); + } + + bool isValid(unsigned S) const override { + return S >= static_cast(System) && + S <= static_cast(Last); + } + + ArrayRef getRuntimeValues() const override { + static_assert(Last == Single, "Does not include all sync scopes"); + static const unsigned Scopes[] = { + static_cast(Device), static_cast(System), + static_cast(Workgroup), static_cast(Wavefront), + static_cast(Single)}; + return llvm::ArrayRef(Scopes); + } + + unsigned getFallBackValue() const override { + return static_cast(System); + } +}; + inline std::unique_ptr AtomicScopeModel::create(AtomicScopeModelKind K) { switch (K) { @@ -214,6 +279,8 @@ AtomicScopeModel::create(AtomicScopeModelKind K) { return std::make_unique(); case AtomicScopeModelKind::HIP: return std::make_unique(); + case AtomicScopeModelKind::Generic: + return std::make_unique(); } llvm_unreachable("Invalid atomic scope model kind"); } diff --git a/clang/include/clang/Basic/TargetInfo.h b/clang/include/clang/Basic/TargetInfo.h index 41f3c2e403cb..ec0189627dfb 100644 --- a/clang/include/clang/Basic/TargetInfo.h +++ b/clang/include/clang/Basic/TargetInfo.h @@ -266,7 +266,6 @@ protected: LLVM_PREFERRED_TYPE(bool) unsigned AllowAMDGPUUnsafeFPAtomics : 1; - LLVM_PREFERRED_TYPE(bool) unsigned ARMCDECoprocMask : 8; unsigned MaxOpenCLWorkGroupSize; diff --git a/clang/include/clang/Basic/TargetOSMacros.def b/clang/include/clang/Basic/TargetOSMacros.def new file mode 100644 index 000000000000..dfc2e033f6fd --- /dev/null +++ b/clang/include/clang/Basic/TargetOSMacros.def @@ -0,0 +1,55 @@ +//===--- TargetOSMacros.def - Target OS macros ------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// This file specifies the predefined TARGET_OS_* conditional macros. +// A target macro `Name` should be defined if `Predicate` evaluates to true. +// The macro expects `const llvm::Triple &Triple` and the class `llvm::Triple` +// to be available for the predicate. +// +//===----------------------------------------------------------------------===// + +#ifndef TARGET_OS +#define TARGET_OS(Name, Predicate) +#endif + +// Windows targets. +TARGET_OS(TARGET_OS_WIN32, Triple.isOSWindows()) +TARGET_OS(TARGET_OS_WINDOWS, Triple.isOSWindows()) + +// Linux target. +TARGET_OS(TARGET_OS_LINUX, Triple.isOSLinux()) + +// Unix target. +TARGET_OS(TARGET_OS_UNIX, Triple.isOSNetBSD() || + Triple.isOSFreeBSD() || + Triple.isOSOpenBSD() || + Triple.isOSSolaris()) + +// Apple (Mac) targets. +TARGET_OS(TARGET_OS_MAC, Triple.isOSDarwin()) +TARGET_OS(TARGET_OS_OSX, Triple.isMacOSX()) +TARGET_OS(TARGET_OS_IPHONE, Triple.isiOS() || Triple.isTvOS() || + Triple.isWatchOS()) +// Triple::isiOS() also includes tvOS +TARGET_OS(TARGET_OS_IOS, Triple.getOS() == llvm::Triple::IOS) +TARGET_OS(TARGET_OS_TV, Triple.isTvOS()) +TARGET_OS(TARGET_OS_WATCH, Triple.isWatchOS()) +TARGET_OS(TARGET_OS_DRIVERKIT, Triple.isDriverKit()) +TARGET_OS(TARGET_OS_MACCATALYST, Triple.isMacCatalystEnvironment()) +TARGET_OS(TARGET_OS_SIMULATOR, Triple.isSimulatorEnvironment()) + +// Deprecated Apple target conditionals. +TARGET_OS(TARGET_OS_EMBEDDED, (Triple.isiOS() || Triple.isTvOS() \ + || Triple.isWatchOS()) \ + && !Triple.isMacCatalystEnvironment() \ + && !Triple.isSimulatorEnvironment()) +TARGET_OS(TARGET_OS_NANO, Triple.isWatchOS()) +TARGET_OS(TARGET_IPHONE_SIMULATOR, Triple.isSimulatorEnvironment()) +TARGET_OS(TARGET_OS_UIKITFORMAC, Triple.isMacCatalystEnvironment()) + +#undef TARGET_OS diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def index 5f9915d21022..3f0e1e1a7d45 100644 --- a/clang/include/clang/Basic/TokenKinds.def +++ b/clang/include/clang/Basic/TokenKinds.def @@ -911,6 +911,11 @@ PRAGMA_ANNOTATION(pragma_fenv_access_ms) // handles them. PRAGMA_ANNOTATION(pragma_fenv_round) +// Annotation for #pragma STDC CX_LIMITED_RANGE +// The lexer produces these so that they only take effect when the parser +// handles them. +PRAGMA_ANNOTATION(pragma_cx_limited_range) + // Annotation for #pragma float_control // The lexer produces these so that they only take effect when the parser // handles them. diff --git a/clang/include/clang/Basic/arm_sve.td b/clang/include/clang/Basic/arm_sve.td index 896c6f6e8033..42ad7da737b4 100644 --- a/clang/include/clang/Basic/arm_sve.td +++ b/clang/include/clang/Basic/arm_sve.td @@ -1296,6 +1296,11 @@ def SVCREATE_3_BF16 : SInst<"svcreate3[_{d}]", "3ddd", "b", MergeNone, "", [IsT def SVCREATE_4_BF16 : SInst<"svcreate4[_{d}]", "4dddd", "b", MergeNone, "", [IsTupleCreate]>; } +let TargetGuard = "sve2p1" in { + def SVCREATE_2_B : SInst<"svcreate2[_{d}]", "2dd", "Pc", MergeNone, "", [IsTupleCreate]>; + def SVCREATE_4_B : SInst<"svcreate4[_{d}]", "4dddd", "Pc", MergeNone, "", [IsTupleCreate]>; +} + //////////////////////////////////////////////////////////////////////////////// // Vector insertion and extraction def SVGET_2 : SInst<"svget2[_{d}]", "d2i", "csilUcUsUiUlhfd", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_1>]>; @@ -1316,6 +1321,13 @@ def SVSET_3_BF16 : SInst<"svset3[_{d}]", "33id", "b", MergeNone, "", [IsTupleSet def SVSET_4_BF16 : SInst<"svset4[_{d}]", "44id", "b", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>; } +let TargetGuard = "sve2p1" in { + def SVGET_2_B : SInst<"svget2[_{d}]", "d2i", "Pc", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_1>]>; + def SVGET_4_B : SInst<"svget4[_{d}]", "d4i", "Pc", MergeNone, "", [IsTupleGet], [ImmCheck<1, ImmCheck0_3>]>; + + def SVSET_2_B : SInst<"svset2[_{d}]", "22id", "Pc", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_1>]>; + def SVSET_4_B : SInst<"svset4[_{d}]", "44id", "Pc", MergeNone, "", [IsTupleSet], [ImmCheck<1, ImmCheck0_3>]>; +} //////////////////////////////////////////////////////////////////////////////// // SVE2 WhileGE/GT let TargetGuard = "sve2" in { @@ -1923,16 +1935,25 @@ def SVBGRP : SInst<"svbgrp[_{d}]", "ddd", "UcUsUiUl", MergeNone, "aarch64_sv def SVBGRP_N : SInst<"svbgrp[_n_{d}]", "dda", "UcUsUiUl", MergeNone, "aarch64_sve_bgrp_x">; } -let TargetGuard = "sve2p1" in { -def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [], []>; +let TargetGuard = "sve2p1|sme" in { +def SVPSEL_B : SInst<"svpsel_lane_b8", "PPPm", "Pc", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_H : SInst<"svpsel_lane_b16", "PPPm", "Ps", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_S : SInst<"svpsel_lane_b32", "PPPm", "Pi", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_D : SInst<"svpsel_lane_b64", "PPPm", "Pl", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_COUNT_ALIAS_B : SInst<"svpsel_lane_c8", "}}Pm", "Pc", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_COUNT_ALIAS_H : SInst<"svpsel_lane_c16", "}}Pm", "Ps", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_COUNT_ALIAS_S : SInst<"svpsel_lane_c32", "}}Pm", "Pi", MergeNone, "", [IsStreamingCompatible], []>; +def SVPSEL_COUNT_ALIAS_D : SInst<"svpsel_lane_c64", "}}Pm", "Pl", MergeNone, "", [IsStreamingCompatible], []>; +} -def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [], [ImmCheck<1, ImmCheck0_3>]>; -def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [], [ImmCheck<1, ImmCheck0_1>]>; +let TargetGuard = "sve2p1|sme2" in { +//FIXME: Replace IsStreamingCompatible with IsStreamingOrHasSVE2p1 when available +def SVPEXT_SINGLE : SInst<"svpext_lane_{d}", "P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_3>]>; +def SVPEXT_X2 : SInst<"svpext_lane_{d}_x2", "2.P}i", "QcQsQiQl", MergeNone, "aarch64_sve_pext_x2", [IsStreamingCompatible], [ImmCheck<1, ImmCheck0_1>]>; +} -def SVPSEL_COUNT_ALIAS_B : SInst<"svpsel_lane_c8", "}}Pm", "Pc", MergeNone, "", [], []>; -def SVPSEL_COUNT_ALIAS_H : SInst<"svpsel_lane_c16", "}}Pm", "Ps", MergeNone, "", [], []>; -def SVPSEL_COUNT_ALIAS_S : SInst<"svpsel_lane_c32", "}}Pm", "Pi", MergeNone, "", [], []>; -def SVPSEL_COUNT_ALIAS_D : SInst<"svpsel_lane_c64", "}}Pm", "Pl", MergeNone, "", [], []>; +let TargetGuard = "sve2p1" in { +def SVFCLAMP : SInst<"svclamp[_{d}]", "dddd", "hfd", MergeNone, "aarch64_sve_fclamp", [], []>; def SVWHILEGE_COUNT : SInst<"svwhilege_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilege_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; def SVWHILEGT_COUNT : SInst<"svwhilegt_{d}", "}lli", "QcQsQiQl", MergeNone, "aarch64_sve_whilegt_{d}", [IsOverloadNone], [ImmCheck<2, ImmCheck2_4_Mul2>]>; @@ -2033,11 +2054,6 @@ let TargetGuard = "sve2p1" in { def SVSCLAMP : SInst<"svclamp[_{d}]", "dddd", "csil", MergeNone, "aarch64_sve_sclamp", [], []>; def SVUCLAMP : SInst<"svclamp[_{d}]", "dddd", "UcUsUiUl", MergeNone, "aarch64_sve_uclamp", [], []>; -def SVPSEL_B : SInst<"svpsel_lane_b8", "PPPm", "Pc", MergeNone, "", [], []>; -def SVPSEL_H : SInst<"svpsel_lane_b16", "PPPm", "Ps", MergeNone, "", [], []>; -def SVPSEL_S : SInst<"svpsel_lane_b32", "PPPm", "Pi", MergeNone, "", [], []>; -def SVPSEL_D : SInst<"svpsel_lane_b64", "PPPm", "Pl", MergeNone, "", [], []>; - def SVCNTP_COUNT : SInst<"svcntp_{d}", "n}i", "QcQsQiQl", MergeNone, "aarch64_sve_cntp_{d}", [IsOverloadNone], [ImmCheck<1, ImmCheck2_4_Mul2>]>; defm SVREVD : SInstZPZ<"svrevd", "csilUcUsUiUl", "aarch64_sve_revd">; @@ -2134,6 +2150,24 @@ let TargetGuard = "sme2" in { def SVURSHL_X2 : SInst<"svrshl[_{d}_x2]", "222", "UcUsUiUl", MergeNone, "aarch64_sve_urshl_x2", [IsStreaming], []>; def SVSRSHL_X4 : SInst<"svrshl[_{d}_x4]", "444", "csil", MergeNone, "aarch64_sve_srshl_x4", [IsStreaming], []>; def SVURSHL_X4 : SInst<"svrshl[_{d}_x4]", "444", "UcUsUiUl", MergeNone, "aarch64_sve_urshl_x4", [IsStreaming], []>; + + def SVQRSHRN_X4 : SInst<"svqrshrn[_n]_{0}[_{d}_x4]", "q4i", "il", MergeNone, "aarch64_sve_sqrshrn_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; + def SVUQRSHRN_X4 : SInst<"svqrshrn[_n]_{0}[_{d}_x4]", "b4i", "UiUl", MergeNone, "aarch64_sve_uqrshrn_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; + + // SQRSHR / UQRSHR + def SVQRSHR_X2 : SInst<"svqrshr[_n]_{0}[_{d}_x2]", "h2i", "i", MergeNone, "aarch64_sve_sqrshr_x2", [IsStreaming], [ImmCheck<1, ImmCheck1_16>]>; + def SVUQRSHR_X2 : SInst<"svqrshr[_n]_{0}[_{d}_x2]", "e2i", "Ui", MergeNone, "aarch64_sve_uqrshr_x2", [IsStreaming], [ImmCheck<1, ImmCheck1_16>]>; + def SVQRSHR_X4 : SInst<"svqrshr[_n]_{0}[_{d}_x4]", "q4i", "il", MergeNone, "aarch64_sve_sqrshr_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; + def SVUQRSHR_X4 : SInst<"svqrshr[_n]_{0}[_{d}_x4]", "b4i", "UiUl", MergeNone, "aarch64_sve_uqrshr_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; + + // SQRSHRU + def SVSQRSHRU_X2 : SInst<"svqrshru[_n]_{0}[_{d}_x2]", "e2i", "i", MergeNone, "aarch64_sve_sqrshru_x2", [IsStreaming], [ImmCheck<1, ImmCheck1_16>]>; + def SVSQRSHRU_X4 : SInst<"svqrshru[_n]_{0}[_{d}_x4]", "b4i", "il", MergeNone, "aarch64_sve_sqrshru_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; + + def SVSQRSHRUN_X4 : SInst<"svqrshrun[_n]_{0}[_{d}_x4]", "b4i", "il", MergeNone, "aarch64_sve_sqrshrun_x4", [IsStreaming], [ImmCheck<1, ImmCheckShiftRight, 0>]>; + + def REINTERPRET_SVBOOL_TO_SVCOUNT : Inst<"svreinterpret[_c]", "}P", "Pc", MergeNone, "", [IsStreamingCompatible], []>; + def REINTERPRET_SVCOUNT_TO_SVBOOL : Inst<"svreinterpret[_b]", "P}", "Pc", MergeNone, "", [IsStreamingCompatible], []>; } let TargetGuard = "sve2p1" in { diff --git a/clang/include/clang/Driver/Multilib.h b/clang/include/clang/Driver/Multilib.h index 6a9533e6dd83..9a2cc9bb1ba1 100644 --- a/clang/include/clang/Driver/Multilib.h +++ b/clang/include/clang/Driver/Multilib.h @@ -130,7 +130,7 @@ public: /// Select compatible variants, \returns false if none are compatible bool select(const Multilib::flags_list &Flags, - llvm::SmallVector &) const; + llvm::SmallVectorImpl &) const; unsigned size() const { return Multilibs.size(); } diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td index 0eec2b352637..25c76cf2ad2c 100644 --- a/clang/include/clang/Driver/Options.td +++ b/clang/include/clang/Driver/Options.td @@ -1010,6 +1010,30 @@ defm offload_uniform_block : BoolFOption<"offload-uniform-block", NegFlag, BothFlags<[], [ClangOption], " that kernels are launched with uniform block sizes (default true for CUDA/HIP and false otherwise)">>; +def fcx_limited_range : Joined<["-"], "fcx-limited-range">, + Group, Visibility<[ClangOption, CC1Option]>, + HelpText<"Basic algebraic expansions of complex arithmetic operations " + "involving are enabled.">; + +def fno_cx_limited_range : Joined<["-"], "fno-cx-limited-range">, + Group, Visibility<[ClangOption, CC1Option]>, + HelpText<"Basic algebraic expansions of complex arithmetic operations " + "involving are disabled.">; + +def fcx_fortran_rules : Joined<["-"], "fcx-fortran-rules">, + Group, Visibility<[ClangOption, CC1Option]>, + HelpText<"Range reduction is enabled for complex arithmetic operations.">; + +def fno_cx_fortran_rules : Joined<["-"], "fno-cx-fortran-rules">, + Group, Visibility<[ClangOption, CC1Option]>, + HelpText<"Range reduction is disabled for complex arithmetic operations.">; + +def complex_range_EQ : Joined<["-"], "complex-range=">, Group, + Visibility<[CC1Option]>, + Values<"full,limited,fortran">, NormalizedValuesScope<"LangOptions">, + NormalizedValues<["CX_Full", "CX_Limited", "CX_Fortran"]>, + MarshallingInfoEnum, "CX_Full">; + // OpenCL-only Options def cl_opt_disable : Flag<["-"], "cl-opt-disable">, Group, Visibility<[ClangOption, CC1Option]>, @@ -1818,6 +1842,9 @@ def fcomment_block_commands : CommaJoined<["-"], "fcomment-block-commands=">, Gr Visibility<[ClangOption, CC1Option]>, HelpText<"Treat each comma separated argument in as a documentation comment block command">, MetaVarName<"">, MarshallingInfoStringVector>; +defm define_target_os_macros : OptInCC1FFlag<"define-target-os-macros", + "Enable", "Disable", " predefined target OS macros", + [ClangOption, CC1Option]>; def fparse_all_comments : Flag<["-"], "fparse-all-comments">, Group, Visibility<[ClangOption, CC1Option]>, MarshallingInfoFlag>; @@ -6351,6 +6378,12 @@ def J : JoinedOrSeparate<["-"], "J">, Group, Alias; +let Visibility = [FlangOption] in { +def no_fortran_main : Flag<["-"], "fno-fortran-main">, + Visibility<[FlangOption]>, Group, + HelpText<"Do not include Fortran_main.a (provided by Flang) when linking">; +} // let Visibility = [ FlangOption ] + //===----------------------------------------------------------------------===// // FC1 Options //===----------------------------------------------------------------------===// diff --git a/clang/include/clang/Frontend/FrontendActions.h b/clang/include/clang/Frontend/FrontendActions.h index 3940e00eeb8d..fcce31ac0590 100644 --- a/clang/include/clang/Frontend/FrontendActions.h +++ b/clang/include/clang/Frontend/FrontendActions.h @@ -151,6 +151,9 @@ class GenerateModuleInterfaceAction : public GenerateModuleAction { private: bool BeginSourceFileAction(CompilerInstance &CI) override; + std::unique_ptr CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) override; + std::unique_ptr CreateOutputFile(CompilerInstance &CI, StringRef InFile) override; }; diff --git a/clang/include/clang/Lex/HeaderSearch.h b/clang/include/clang/Lex/HeaderSearch.h index a100598c8015..a2c33842924b 100644 --- a/clang/include/clang/Lex/HeaderSearch.h +++ b/clang/include/clang/Lex/HeaderSearch.h @@ -247,16 +247,13 @@ class HeaderSearch { /// \#include search path information. Requests for \#include "x" search the /// directory of the \#including file first, then each directory in SearchDirs /// consecutively. Requests for search the current dir first, then each - /// directory in SearchDirs, starting at AngledDirIdx, consecutively. If - /// NoCurDirSearch is true, then the check for the file in the current - /// directory is suppressed. + /// directory in SearchDirs, starting at AngledDirIdx, consecutively. std::vector SearchDirs; /// Whether the DirectoryLookup at the corresponding index in SearchDirs has /// been successfully used to lookup a file. std::vector SearchDirsUsage; unsigned AngledDirIdx = 0; unsigned SystemDirIdx = 0; - bool NoCurDirSearch = false; /// Maps HeaderMap keys to SearchDir indices. When HeaderMaps are used /// heavily, SearchDirs can start with thousands of HeaderMaps, so this Index @@ -373,7 +370,7 @@ public: /// Interface for setting the file search paths. void SetSearchPaths(std::vector dirs, unsigned angledDirIdx, - unsigned systemDirIdx, bool noCurDirSearch, + unsigned systemDirIdx, llvm::DenseMap searchDirToHSEntry); /// Add an additional search path. diff --git a/clang/include/clang/Lex/ModuleMap.h b/clang/include/clang/Lex/ModuleMap.h index 32e7e8f899e5..867cb6eab42f 100644 --- a/clang/include/clang/Lex/ModuleMap.h +++ b/clang/include/clang/Lex/ModuleMap.h @@ -82,7 +82,7 @@ class ModuleMap { /// The directory used for Clang-supplied, builtin include headers, /// such as "stdint.h". - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr BuiltinIncludeDir; + OptionalDirectoryEntryRef BuiltinIncludeDir; /// Language options used to parse the module map itself. /// @@ -408,16 +408,12 @@ public: /// Set the target information. void setTarget(const TargetInfo &Target); - /// Set the directory that contains Clang-supplied include - /// files, such as our stdarg.h or tgmath.h. - void setBuiltinIncludeDir(DirectoryEntryRef Dir) { - BuiltinIncludeDir = Dir; - } + /// Set the directory that contains Clang-supplied include files, such as our + /// stdarg.h or tgmath.h. + void setBuiltinIncludeDir(DirectoryEntryRef Dir) { BuiltinIncludeDir = Dir; } /// Get the directory that contains Clang-supplied include files. - OptionalDirectoryEntryRefDegradesToDirectoryEntryPtr getBuiltinDir() const { - return BuiltinIncludeDir; - } + OptionalDirectoryEntryRef getBuiltinDir() const { return BuiltinIncludeDir; } /// Is this a compiler builtin header? bool isBuiltinHeader(FileEntryRef File); diff --git a/clang/include/clang/Lex/PreprocessorLexer.h b/clang/include/clang/Lex/PreprocessorLexer.h index eebaad7d50db..d71fe708ab20 100644 --- a/clang/include/clang/Lex/PreprocessorLexer.h +++ b/clang/include/clang/Lex/PreprocessorLexer.h @@ -157,7 +157,7 @@ public: /// getFileEntry - Return the FileEntry corresponding to this FileID. Like /// getFileID(), this only works for lexers with attached preprocessors. - OptionalFileEntryRefDegradesToFileEntryPtr getFileEntry() const; + OptionalFileEntryRef getFileEntry() const; /// Iterator that traverses the current stack of preprocessor /// conditional directives (\#if/\#ifdef/\#ifndef). diff --git a/clang/include/clang/Lex/PreprocessorOptions.h b/clang/include/clang/Lex/PreprocessorOptions.h index 058194bcde72..f841e4a028df 100644 --- a/clang/include/clang/Lex/PreprocessorOptions.h +++ b/clang/include/clang/Lex/PreprocessorOptions.h @@ -76,6 +76,9 @@ public: /// predefines. bool UsePredefines = true; + /// Indicates whether to predefine target OS macros. + bool DefineTargetOSMacros = false; + /// Whether we should maintain a detailed record of all macro /// definitions and expansions. bool DetailedRecord = false; diff --git a/clang/include/clang/Parse/Parser.h b/clang/include/clang/Parse/Parser.h index 32d0b76c35b0..2dbe090bd093 100644 --- a/clang/include/clang/Parse/Parser.h +++ b/clang/include/clang/Parse/Parser.h @@ -769,6 +769,10 @@ private: /// #pragma STDC FENV_ROUND... void HandlePragmaFEnvRound(); + /// Handle the annotation token produced for + /// #pragma STDC CX_LIMITED_RANGE... + void HandlePragmaCXLimitedRange(); + /// Handle the annotation token produced for /// #pragma float_control void HandlePragmaFloatControl(); @@ -3544,6 +3548,7 @@ private: void ParseOpenACCCacheVarList(); /// Parses a single variable in a variable list for the 'cache' construct. bool ParseOpenACCCacheVar(); + bool ParseOpenACCWaitArgument(); private: //===--------------------------------------------------------------------===// diff --git a/clang/include/clang/Sema/HLSLExternalSemaSource.h b/clang/include/clang/Sema/HLSLExternalSemaSource.h index 4b6bc96f72e2..c0bfff327139 100644 --- a/clang/include/clang/Sema/HLSLExternalSemaSource.h +++ b/clang/include/clang/Sema/HLSLExternalSemaSource.h @@ -30,9 +30,9 @@ class HLSLExternalSemaSource : public ExternalSemaSource { void defineHLSLVectorAlias(); void defineTrivialHLSLTypes(); - void forwardDeclareHLSLTypes(); + void defineHLSLTypesWithForwardDeclarations(); - void completeBufferType(CXXRecordDecl *Record); + void onCompletion(CXXRecordDecl *Record, CompletionFunction Fn); public: ~HLSLExternalSemaSource() override; diff --git a/clang/include/clang/Sema/Sema.h b/clang/include/clang/Sema/Sema.h index 26f4e38a46e8..1902d098f3c2 100644 --- a/clang/include/clang/Sema/Sema.h +++ b/clang/include/clang/Sema/Sema.h @@ -8561,6 +8561,10 @@ public: TemplateParameterList *Params, TemplateArgumentLoc &Arg); + void NoteTemplateLocation(const NamedDecl &Decl, + std::optional ParamRange = {}); + void NoteTemplateParameterLocation(const NamedDecl &Decl); + ExprResult BuildExpressionFromDeclTemplateArgument(const TemplateArgument &Arg, QualType ParamType, @@ -11019,6 +11023,11 @@ public: /// \#pragma STDC FENV_ACCESS void ActOnPragmaFEnvAccess(SourceLocation Loc, bool IsEnabled); + /// ActOnPragmaCXLimitedRange - Called on well formed + /// \#pragma STDC CX_LIMITED_RANGE + void ActOnPragmaCXLimitedRange(SourceLocation Loc, + LangOptions::ComplexRangeKind Range); + /// Called on well formed '\#pragma clang fp' that has option 'exceptions'. void ActOnPragmaFPExceptions(SourceLocation Loc, LangOptions::FPExceptionModeKind); diff --git a/clang/include/clang/Serialization/ASTReader.h b/clang/include/clang/Serialization/ASTReader.h index 7eefdca6815c..9bb89ec94109 100644 --- a/clang/include/clang/Serialization/ASTReader.h +++ b/clang/include/clang/Serialization/ASTReader.h @@ -2415,12 +2415,7 @@ public: BitsUnpacker(BitsUnpacker &&) = delete; BitsUnpacker operator=(const BitsUnpacker &) = delete; BitsUnpacker operator=(BitsUnpacker &&) = delete; - ~BitsUnpacker() { -#ifndef NDEBUG - while (isValid()) - assert(!getNextBit() && "There are unprocessed bits!"); -#endif - } + ~BitsUnpacker() = default; void updateValue(uint32_t V) { Value = V; diff --git a/clang/include/clang/Serialization/ModuleFile.h b/clang/include/clang/Serialization/ModuleFile.h index 48be8676cc26..9a14129d72ff 100644 --- a/clang/include/clang/Serialization/ModuleFile.h +++ b/clang/include/clang/Serialization/ModuleFile.h @@ -104,7 +104,7 @@ public: return File; } - OptionalFileEntryRefDegradesToFileEntryPtr getFile() const { + OptionalFileEntryRef getFile() const { if (auto *P = Val.getPointer()) return FileEntryRef(*P); return std::nullopt; @@ -123,8 +123,8 @@ public: /// other modules. class ModuleFile { public: - ModuleFile(ModuleKind Kind, unsigned Generation) - : Kind(Kind), Generation(Generation) {} + ModuleFile(ModuleKind Kind, FileEntryRef File, unsigned Generation) + : Kind(Kind), File(File), Generation(Generation) {} ~ModuleFile(); // === General information === @@ -176,7 +176,7 @@ public: bool DidReadTopLevelSubmodule = false; /// The file entry for the module file. - OptionalFileEntryRefDegradesToFileEntryPtr File; + FileEntryRef File; /// The signature of the module file, which may be used instead of the size /// and modification time to identify this particular file. diff --git a/clang/lib/AST/Decl.cpp b/clang/lib/AST/Decl.cpp index c5c2edf1bfe3..527ea6042daa 100644 --- a/clang/lib/AST/Decl.cpp +++ b/clang/lib/AST/Decl.cpp @@ -4150,6 +4150,7 @@ FunctionDecl::setFunctionTemplateSpecialization(ASTContext &C, assert(TSK != TSK_Undeclared && "Must specify the type of function template specialization"); assert((TemplateOrSpecialization.isNull() || + getFriendObjectKind() != FOK_None || TSK == TSK_ExplicitSpecialization) && "Member specialization must be an explicit specialization"); FunctionTemplateSpecializationInfo *Info = diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp index 55c6b732b708..b125fc676da8 100644 --- a/clang/lib/AST/Expr.cpp +++ b/clang/lib/AST/Expr.cpp @@ -4887,6 +4887,7 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) { case AO__atomic_load_n: return 2; + case AO__scoped_atomic_load_n: case AO__opencl_atomic_load: case AO__hip_atomic_load: case AO__c11_atomic_store: @@ -4921,6 +4922,26 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) { case AO__atomic_fetch_max: return 3; + case AO__scoped_atomic_load: + case AO__scoped_atomic_store: + case AO__scoped_atomic_store_n: + case AO__scoped_atomic_fetch_add: + case AO__scoped_atomic_fetch_sub: + case AO__scoped_atomic_fetch_and: + case AO__scoped_atomic_fetch_or: + case AO__scoped_atomic_fetch_xor: + case AO__scoped_atomic_fetch_nand: + case AO__scoped_atomic_add_fetch: + case AO__scoped_atomic_sub_fetch: + case AO__scoped_atomic_and_fetch: + case AO__scoped_atomic_or_fetch: + case AO__scoped_atomic_xor_fetch: + case AO__scoped_atomic_nand_fetch: + case AO__scoped_atomic_min_fetch: + case AO__scoped_atomic_max_fetch: + case AO__scoped_atomic_fetch_min: + case AO__scoped_atomic_fetch_max: + case AO__scoped_atomic_exchange_n: case AO__hip_atomic_exchange: case AO__hip_atomic_fetch_add: case AO__hip_atomic_fetch_sub: @@ -4942,6 +4963,7 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) { case AO__atomic_exchange: return 4; + case AO__scoped_atomic_exchange: case AO__c11_atomic_compare_exchange_strong: case AO__c11_atomic_compare_exchange_weak: return 5; @@ -4952,6 +4974,10 @@ unsigned AtomicExpr::getNumSubExprs(AtomicOp Op) { case AO__atomic_compare_exchange: case AO__atomic_compare_exchange_n: return 6; + + case AO__scoped_atomic_compare_exchange: + case AO__scoped_atomic_compare_exchange_n: + return 7; } llvm_unreachable("unknown atomic op"); } diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp index 986302e1fd22..f035c1419f4c 100644 --- a/clang/lib/AST/ExprConstant.cpp +++ b/clang/lib/AST/ExprConstant.cpp @@ -641,6 +641,10 @@ namespace { return false; } + /// Whether we're in a context where [[msvc::constexpr]] evaluation is + /// permitted. See MSConstexprDocs for description of permitted contexts. + bool CanEvalMSConstexpr = false; + private: APValue &createLocal(APValue::LValueBase Base, const void *Key, QualType T, ScopeKind Scope); @@ -674,6 +678,19 @@ namespace { private: llvm::TimeTraceScope TimeScope; }; + + /// RAII object used to change the current ability of + /// [[msvc::constexpr]] evaulation. + struct MSConstexprContextRAII { + CallStackFrame &Frame; + bool OldValue; + explicit MSConstexprContextRAII(CallStackFrame &Frame, bool Value) + : Frame(Frame), OldValue(Frame.CanEvalMSConstexpr) { + Frame.CanEvalMSConstexpr = Value; + } + + ~MSConstexprContextRAII() { Frame.CanEvalMSConstexpr = OldValue; } + }; } static bool HandleDestruction(EvalInfo &Info, const Expr *E, @@ -5546,11 +5563,14 @@ static EvalStmtResult EvaluateStmt(StmtResult &Result, EvalInfo &Info, case Stmt::LabelStmtClass: return EvaluateStmt(Result, Info, cast(S)->getSubStmt(), Case); - case Stmt::AttributedStmtClass: - // As a general principle, C++11 attributes can be ignored without - // any semantic impact. - return EvaluateStmt(Result, Info, cast(S)->getSubStmt(), - Case); + case Stmt::AttributedStmtClass: { + const auto *AS = cast(S); + const auto *SS = AS->getSubStmt(); + MSConstexprContextRAII ConstexprContext( + *Info.CurrentCall, hasSpecificAttr(AS->getAttrs()) && + isa(SS)); + return EvaluateStmt(Result, Info, SS, Case); + } case Stmt::CaseStmtClass: case Stmt::DefaultStmtClass: @@ -5621,7 +5641,9 @@ static bool CheckConstexprFunction(EvalInfo &Info, SourceLocation CallLoc, } // Can we evaluate this function call? - if (Definition && Definition->isConstexpr() && Body) + if (Definition && Body && + (Definition->isConstexpr() || (Info.CurrentCall->CanEvalMSConstexpr && + Definition->hasAttr()))) return true; if (Info.getLangOpts().CPlusPlus11) { diff --git a/clang/lib/AST/Interp/ByteCodeExprGen.cpp b/clang/lib/AST/Interp/ByteCodeExprGen.cpp index f45e8624a774..f7f8e6c73d84 100644 --- a/clang/lib/AST/Interp/ByteCodeExprGen.cpp +++ b/clang/lib/AST/Interp/ByteCodeExprGen.cpp @@ -15,7 +15,6 @@ #include "Function.h" #include "PrimType.h" #include "Program.h" -#include "State.h" using namespace clang; using namespace clang::interp; diff --git a/clang/lib/AST/Interp/Interp.h b/clang/lib/AST/Interp/Interp.h index 4f7778bdd2ff..6cf122f2ba55 100644 --- a/clang/lib/AST/Interp/Interp.h +++ b/clang/lib/AST/Interp/Interp.h @@ -1619,7 +1619,11 @@ bool CastFloatingIntegral(InterpState &S, CodePtr OpPC) { QualType Type = E->getType(); S.CCEDiag(E, diag::note_constexpr_overflow) << F.getAPFloat() << Type; - return S.noteUndefinedBehavior(); + if (S.noteUndefinedBehavior()) { + S.Stk.push(T(Result)); + return true; + } + return false; } S.Stk.push(T(Result)); diff --git a/clang/lib/AST/Interp/InterpBuiltin.cpp b/clang/lib/AST/Interp/InterpBuiltin.cpp index 9cf206ecc212..4384ace6b6be 100644 --- a/clang/lib/AST/Interp/InterpBuiltin.cpp +++ b/clang/lib/AST/Interp/InterpBuiltin.cpp @@ -34,6 +34,19 @@ PrimType getIntPrimType(const InterpState &S) { llvm_unreachable("Int isn't 16 or 32 bit?"); } +PrimType getLongPrimType(const InterpState &S) { + const TargetInfo &TI = S.getCtx().getTargetInfo(); + unsigned LongWidth = TI.getLongWidth(); + + if (LongWidth == 64) + return PT_Sint64; + else if (LongWidth == 32) + return PT_Sint32; + else if (LongWidth == 16) + return PT_Sint16; + llvm_unreachable("long isn't 16, 32 or 64 bit?"); +} + /// Peek an integer value from the stack into an APSInt. static APSInt peekToAPSInt(InterpStack &Stk, PrimType T, size_t Offset = 0) { if (Offset == 0) @@ -110,6 +123,19 @@ static void pushAPSInt(InterpState &S, const APSInt &Val) { } } +/// Pushes \p Val to the stack, as a target-dependent 'long'. +static void pushLong(InterpState &S, int64_t Val) { + PrimType LongType = getLongPrimType(S); + if (LongType == PT_Sint64) + S.Stk.push>(Integral<64, true>::from(Val)); + else if (LongType == PT_Sint32) + S.Stk.push>(Integral<32, true>::from(Val)); + else if (LongType == PT_Sint16) + S.Stk.push>(Integral<16, true>::from(Val)); + else + llvm_unreachable("Long isn't 16, 32 or 64 bit?"); +} + static void pushSizeT(InterpState &S, uint64_t Val) { const TargetInfo &TI = S.getCtx().getTargetInfo(); unsigned SizeTWidth = TI.getTypeWidth(TI.getSizeType()); @@ -533,6 +559,26 @@ static bool interp__builtin_classify_type(InterpState &S, CodePtr OpPC, return true; } +// __builtin_expect(long, long) +// __builtin_expect_with_probability(long, long, double) +static bool interp__builtin_expect(InterpState &S, CodePtr OpPC, + const InterpFrame *Frame, + const Function *Func, const CallExpr *Call) { + // The return value is simply the value of the first parameter. + // We ignore the probability. + unsigned NumArgs = Call->getNumArgs(); + assert(NumArgs == 2 || NumArgs == 3); + + PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType()); + unsigned Offset = align(primSize(getLongPrimType(S))) * 2; + if (NumArgs == 3) + Offset += align(primSize(PT_Float)); + + APSInt Val = peekToAPSInt(S.Stk, ArgT, Offset); + pushLong(S, Val.getSExtValue()); + return true; +} + bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F, const CallExpr *Call) { InterpFrame *Frame = S.Current; @@ -702,6 +748,12 @@ bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const Function *F, return false; break; + case Builtin::BI__builtin_expect: + case Builtin::BI__builtin_expect_with_probability: + if (!interp__builtin_expect(S, OpPC, Frame, F, Call)) + return false; + break; + default: return false; } diff --git a/clang/lib/AST/Interp/InterpFrame.cpp b/clang/lib/AST/Interp/InterpFrame.cpp index b06923114c7a..d460d7ea3710 100644 --- a/clang/lib/AST/Interp/InterpFrame.cpp +++ b/clang/lib/AST/Interp/InterpFrame.cpp @@ -228,7 +228,7 @@ Pointer InterpFrame::getParamPointer(unsigned Off) { SourceInfo InterpFrame::getSource(CodePtr PC) const { // Implicitly created functions don't have any code we could point at, // so return the call site. - if (Func && Func->getDecl()->isImplicit() && Caller) + if (Func && (!Func->hasBody() || Func->getDecl()->isImplicit()) && Caller) return Caller->getSource(RetPC); return S.getSource(Func, PC); @@ -243,7 +243,7 @@ SourceLocation InterpFrame::getLocation(CodePtr PC) const { } SourceRange InterpFrame::getRange(CodePtr PC) const { - if (Func && Func->getDecl()->isImplicit() && Caller) + if (Func && (!Func->hasBody() || Func->getDecl()->isImplicit()) && Caller) return Caller->getRange(RetPC); return S.getRange(Func, PC); diff --git a/clang/lib/AST/MicrosoftMangle.cpp b/clang/lib/AST/MicrosoftMangle.cpp index 50ab6ea59be9..c59a66e103a6 100644 --- a/clang/lib/AST/MicrosoftMangle.cpp +++ b/clang/lib/AST/MicrosoftMangle.cpp @@ -3809,14 +3809,14 @@ void MicrosoftMangleContextImpl::mangleCXXRTTICompleteObjectLocator( llvm::raw_svector_ostream Stream(VFTableMangling); mangleCXXVFTable(Derived, BasePath, Stream); - if (VFTableMangling.startswith("??@")) { - assert(VFTableMangling.endswith("@")); + if (VFTableMangling.starts_with("??@")) { + assert(VFTableMangling.ends_with("@")); Out << VFTableMangling << "??_R4@"; return; } - assert(VFTableMangling.startswith("??_7") || - VFTableMangling.startswith("??_S")); + assert(VFTableMangling.starts_with("??_7") || + VFTableMangling.starts_with("??_S")); Out << "??_R4" << VFTableMangling.str().drop_front(4); } diff --git a/clang/lib/AST/RecordLayoutBuilder.cpp b/clang/lib/AST/RecordLayoutBuilder.cpp index a51c8b938f41..706991f4fb50 100644 --- a/clang/lib/AST/RecordLayoutBuilder.cpp +++ b/clang/lib/AST/RecordLayoutBuilder.cpp @@ -2942,8 +2942,8 @@ void MicrosoftRecordLayoutBuilder::layoutNonVirtualBase( } if (!FoundBase) { - if (MDCUsesEBO && BaseDecl->isEmpty()) { - assert(BaseLayout.getNonVirtualSize() == CharUnits::Zero()); + if (MDCUsesEBO && BaseDecl->isEmpty() && + (BaseLayout.getNonVirtualSize() == CharUnits::Zero())) { BaseOffset = CharUnits::Zero(); } else { // Otherwise, lay the base out at the end of the MDC. diff --git a/clang/lib/AST/StmtPrinter.cpp b/clang/lib/AST/StmtPrinter.cpp index ab4a013de5f5..c04cb313c338 100644 --- a/clang/lib/AST/StmtPrinter.cpp +++ b/clang/lib/AST/StmtPrinter.cpp @@ -1841,6 +1841,7 @@ void StmtPrinter::VisitAtomicExpr(AtomicExpr *Node) { PrintExpr(Node->getPtr()); if (Node->getOp() != AtomicExpr::AO__c11_atomic_load && Node->getOp() != AtomicExpr::AO__atomic_load_n && + Node->getOp() != AtomicExpr::AO__scoped_atomic_load_n && Node->getOp() != AtomicExpr::AO__opencl_atomic_load && Node->getOp() != AtomicExpr::AO__hip_atomic_load) { OS << ", "; diff --git a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp index 7430ef599e03..2a7bfce53501 100644 --- a/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp +++ b/clang/lib/Analysis/FlowSensitive/HTMLLogger.cpp @@ -351,9 +351,10 @@ private: if (Invalid) return; - static constexpr unsigned Missing = -1; // TokenInfo stores the BB and set of elements that a token is part of. struct TokenInfo { + enum : unsigned { Missing = static_cast(-1) }; + // The basic block this is part of. // This is the BB of the stmt with the smallest containing range. unsigned BB = Missing; diff --git a/clang/lib/Analysis/ThreadSafety.cpp b/clang/lib/Analysis/ThreadSafety.cpp index 7fdf22c2f391..e25b843c9bf8 100644 --- a/clang/lib/Analysis/ThreadSafety.cpp +++ b/clang/lib/Analysis/ThreadSafety.cpp @@ -1010,6 +1010,8 @@ class ThreadSafetyAnalyzer { ThreadSafetyHandler &Handler; const FunctionDecl *CurrentFunction; LocalVariableMap LocalVarMap; + // Maps constructed objects to `this` placeholder prior to initialization. + llvm::SmallDenseMap ConstructedObjects; FactManager FactMan; std::vector BlockInfo; @@ -1543,8 +1545,6 @@ class BuildLockset : public ConstStmtVisitor { FactSet FSet; // The fact set for the function on exit. const FactSet &FunctionExitFSet; - /// Maps constructed objects to `this` placeholder prior to initialization. - llvm::SmallDenseMap ConstructedObjects; LocalVariableMap::Context LVarCtx; unsigned CtxIndex; @@ -1808,7 +1808,7 @@ void BuildLockset::handleCall(const Expr *Exp, const NamedDecl *D, std::pair Placeholder = Analyzer->SxBuilder.createThisPlaceholder(Exp); [[maybe_unused]] auto inserted = - ConstructedObjects.insert({Exp, Placeholder.first}); + Analyzer->ConstructedObjects.insert({Exp, Placeholder.first}); assert(inserted.second && "Are we visiting the same expression again?"); if (isa(Exp)) Self = Placeholder.first; @@ -2128,10 +2128,10 @@ void BuildLockset::VisitDeclStmt(const DeclStmt *S) { E = EWC->getSubExpr()->IgnoreParens(); E = UnpackConstruction(E); - if (auto Object = ConstructedObjects.find(E); - Object != ConstructedObjects.end()) { + if (auto Object = Analyzer->ConstructedObjects.find(E); + Object != Analyzer->ConstructedObjects.end()) { Object->second->setClangDecl(VD); - ConstructedObjects.erase(Object); + Analyzer->ConstructedObjects.erase(Object); } } } @@ -2140,11 +2140,11 @@ void BuildLockset::VisitDeclStmt(const DeclStmt *S) { void BuildLockset::VisitMaterializeTemporaryExpr( const MaterializeTemporaryExpr *Exp) { if (const ValueDecl *ExtD = Exp->getExtendingDecl()) { - if (auto Object = - ConstructedObjects.find(UnpackConstruction(Exp->getSubExpr())); - Object != ConstructedObjects.end()) { + if (auto Object = Analyzer->ConstructedObjects.find( + UnpackConstruction(Exp->getSubExpr())); + Object != Analyzer->ConstructedObjects.end()) { Object->second->setClangDecl(ExtD); - ConstructedObjects.erase(Object); + Analyzer->ConstructedObjects.erase(Object); } } } @@ -2487,15 +2487,15 @@ void ThreadSafetyAnalyzer::runAnalysis(AnalysisDeclContext &AC) { // Clean up constructed object even if there are no attributes to // keep the number of objects in limbo as small as possible. - if (auto Object = LocksetBuilder.ConstructedObjects.find( + if (auto Object = ConstructedObjects.find( TD.getBindTemporaryExpr()->getSubExpr()); - Object != LocksetBuilder.ConstructedObjects.end()) { + Object != ConstructedObjects.end()) { const auto *DD = TD.getDestructorDecl(AC.getASTContext()); if (DD->hasAttrs()) // TODO: the location here isn't quite correct. LocksetBuilder.handleCall(nullptr, DD, Object->second, TD.getBindTemporaryExpr()->getEndLoc()); - LocksetBuilder.ConstructedObjects.erase(Object); + ConstructedObjects.erase(Object); } break; } diff --git a/clang/lib/Analysis/UninitializedValues.cpp b/clang/lib/Analysis/UninitializedValues.cpp index b796f7674cc1..e9111ded64eb 100644 --- a/clang/lib/Analysis/UninitializedValues.cpp +++ b/clang/lib/Analysis/UninitializedValues.cpp @@ -64,7 +64,7 @@ static bool isTrackedVar(const VarDecl *vd, const DeclContext *dc) { QualType ty = vd->getType(); if (const auto *RD = ty->getAsRecordDecl()) return recordIsNotEmpty(RD); - return ty->isScalarType() || ty->isVectorType() || ty->isRVVType(); + return ty->isScalarType() || ty->isVectorType() || ty->isRVVSizelessBuiltinType(); } return false; } diff --git a/clang/lib/Analysis/UnsafeBufferUsage.cpp b/clang/lib/Analysis/UnsafeBufferUsage.cpp index e332a3609290..a1efb76be68b 100644 --- a/clang/lib/Analysis/UnsafeBufferUsage.cpp +++ b/clang/lib/Analysis/UnsafeBufferUsage.cpp @@ -1028,6 +1028,46 @@ public: } }; +// Representing a pointer type expression of the form `Ptr += n` in an +// Unspecified Untyped Context (UUC): +class UUCAddAssignGadget : public FixableGadget { +private: + static constexpr const char *const UUCAddAssignTag = + "PointerAddAssignUnderUUC"; + static constexpr const char *const OffsetTag = "Offset"; + + const BinaryOperator *Node; // the `Ptr += n` node + const Expr *Offset = nullptr; + +public: + UUCAddAssignGadget(const MatchFinder::MatchResult &Result) + : FixableGadget(Kind::UUCAddAssign), + Node(Result.Nodes.getNodeAs(UUCAddAssignTag)), + Offset(Result.Nodes.getNodeAs(OffsetTag)) { + assert(Node != nullptr && "Expecting a non-null matching result"); + } + + static bool classof(const Gadget *G) { + return G->getKind() == Kind::UUCAddAssign; + } + + static Matcher matcher() { + return stmt(isInUnspecifiedUntypedContext(expr(ignoringImpCasts( + binaryOperator(hasOperatorName("+="), + hasLHS(declRefExpr(toSupportedVariable())), + hasRHS(expr().bind(OffsetTag))) + .bind(UUCAddAssignTag))))); + } + + virtual std::optional getFixits(const Strategy &S) const override; + + virtual const Stmt *getBaseStmt() const override { return Node; } + + virtual DeclUseList getClaimedVarUseSites() const override { + return {dyn_cast(Node->getLHS())}; + } +}; + // Representing a fixable expression of the form `*(ptr + 123)` or `*(123 + // ptr)`: class DerefSimplePtrArithFixableGadget : public FixableGadget { @@ -1312,6 +1352,16 @@ PointerInitGadget::getFixits(const Strategy &S) const { return std::nullopt; } +static bool isNonNegativeIntegerExpr(const Expr *Expr, const VarDecl *VD, + const ASTContext &Ctx) { + if (auto ConstVal = Expr->getIntegerConstantExpr(Ctx)) { + if (ConstVal->isNegative()) + return false; + } else if (!Expr->getType()->isUnsignedIntegerType()) + return false; + return true; +} + std::optional ULCArraySubscriptGadget::getFixits(const Strategy &S) const { if (const auto *DRE = @@ -1319,14 +1369,12 @@ ULCArraySubscriptGadget::getFixits(const Strategy &S) const { if (const auto *VD = dyn_cast(DRE->getDecl())) { switch (S.lookup(VD)) { case Strategy::Kind::Span: { + // If the index has a negative constant value, we give up as no valid // fix-it can be generated: const ASTContext &Ctx = // FIXME: we need ASTContext to be passed in! VD->getASTContext(); - if (auto ConstVal = Node->getIdx()->getIntegerConstantExpr(Ctx)) { - if (ConstVal->isNegative()) - return std::nullopt; - } else if (!Node->getIdx()->getType()->isUnsignedIntegerType()) + if (!isNonNegativeIntegerExpr(Node->getIdx(), VD, Ctx)) return std::nullopt; // no-op is a good fix-it, otherwise return FixItList{}; @@ -1405,10 +1453,8 @@ static std::optional getPastLoc(const NodeTy *Node, const LangOptions &LangOpts) { SourceLocation Loc = Lexer::getLocForEndOfToken(Node->getEndLoc(), 0, SM, LangOpts); - if (Loc.isValid()) return Loc; - return std::nullopt; } @@ -1766,6 +1812,47 @@ fixUPCAddressofArraySubscriptWithSpan(const UnaryOperator *Node) { FixItHint::CreateReplacement(Node->getSourceRange(), SS.str())}; } +std::optional +UUCAddAssignGadget::getFixits(const Strategy &S) const { + DeclUseList DREs = getClaimedVarUseSites(); + + if (DREs.size() != 1) + return std::nullopt; // In cases of `Ptr += n` where `Ptr` is not a DRE, we + // give up + if (const VarDecl *VD = dyn_cast(DREs.front()->getDecl())) { + if (S.lookup(VD) == Strategy::Kind::Span) { + FixItList Fixes; + + const Stmt *AddAssignNode = getBaseStmt(); + StringRef varName = VD->getName(); + const ASTContext &Ctx = VD->getASTContext(); + + if (!isNonNegativeIntegerExpr(Offset, VD, Ctx)) + return std::nullopt; + + // To transform UUC(p += n) to UUC(p = p.subspan(..)): + bool NotParenExpr = + (Offset->IgnoreParens()->getBeginLoc() == Offset->getBeginLoc()); + std::string SS = varName.str() + " = " + varName.str() + ".subspan"; + if (NotParenExpr) + SS += "("; + + std::optional AddAssignLocation = getEndCharLoc( + AddAssignNode, Ctx.getSourceManager(), Ctx.getLangOpts()); + if (!AddAssignLocation) + return std::nullopt; + + Fixes.push_back(FixItHint::CreateReplacement( + SourceRange(AddAssignNode->getBeginLoc(), Node->getOperatorLoc()), + SS)); + if (NotParenExpr) + Fixes.push_back(FixItHint::CreateInsertion( + Offset->getEndLoc().getLocWithOffset(1), ")")); + return Fixes; + } + } + return std::nullopt; // Not in the cases that we can handle for now, give up. +} std::optional UPCPreIncrementGadget::getFixits(const Strategy &S) const { DeclUseList DREs = getClaimedVarUseSites(); diff --git a/clang/lib/Basic/Cuda.cpp b/clang/lib/Basic/Cuda.cpp index 65840b9f2025..1b1da6a1356f 100644 --- a/clang/lib/Basic/Cuda.cpp +++ b/clang/lib/Basic/Cuda.cpp @@ -39,6 +39,8 @@ static const CudaVersionMapEntry CudaNameVersionMap[] = { CUDA_ENTRY(11, 8), CUDA_ENTRY(12, 0), CUDA_ENTRY(12, 1), + CUDA_ENTRY(12, 2), + CUDA_ENTRY(12, 3), {"", CudaVersion::NEW, llvm::VersionTuple(std::numeric_limits::max())}, {"unknown", CudaVersion::UNKNOWN, {}} // End of list tombstone. }; @@ -93,6 +95,7 @@ static const CudaArchToStringMap arch_names[] = { SM(87), // Jetson/Drive AGX Orin SM(89), // Ada Lovelace SM(90), // Hopper + SM(90a), // Hopper GFX(600), // gfx600 GFX(601), // gfx601 GFX(602), // gfx602 @@ -209,6 +212,8 @@ CudaVersion MinVersionForCudaArch(CudaArch A) { case CudaArch::SM_89: case CudaArch::SM_90: return CudaVersion::CUDA_118; + case CudaArch::SM_90a: + return CudaVersion::CUDA_120; default: llvm_unreachable("invalid enum"); } diff --git a/clang/lib/Basic/Module.cpp b/clang/lib/Basic/Module.cpp index e4ac1abf12a7..7523e509a471 100644 --- a/clang/lib/Basic/Module.cpp +++ b/clang/lib/Basic/Module.cpp @@ -89,7 +89,7 @@ static bool isPlatformEnvironment(const TargetInfo &Target, StringRef Feature) { // where both are valid examples of the same platform+environment but in the // variant (2) the simulator is hardcoded as part of the platform name. Both // forms above should match for "iossimulator" requirement. - if (Target.getTriple().isOSDarwin() && PlatformEnv.endswith("simulator")) + if (Target.getTriple().isOSDarwin() && PlatformEnv.ends_with("simulator")) return PlatformEnv == Feature || CmpPlatformEnv(PlatformEnv, Feature); return PlatformEnv == Feature; diff --git a/clang/lib/Basic/Targets/AMDGPU.cpp b/clang/lib/Basic/Targets/AMDGPU.cpp index 409ae32ab424..719fc51bfc28 100644 --- a/clang/lib/Basic/Targets/AMDGPU.cpp +++ b/clang/lib/Basic/Targets/AMDGPU.cpp @@ -37,50 +37,50 @@ static const char *const DataLayoutStringAMDGCN = "-ni:7:8"; const LangASMap AMDGPUTargetInfo::AMDGPUDefIsGenMap = { - Generic, // Default - Global, // opencl_global - Local, // opencl_local - Constant, // opencl_constant - Private, // opencl_private - Generic, // opencl_generic - Global, // opencl_global_device - Global, // opencl_global_host - Global, // cuda_device - Constant, // cuda_constant - Local, // cuda_shared - Global, // sycl_global - Global, // sycl_global_device - Global, // sycl_global_host - Local, // sycl_local - Private, // sycl_private - Generic, // ptr32_sptr - Generic, // ptr32_uptr - Generic, // ptr64 - Generic, // hlsl_groupshared + llvm::AMDGPUAS::FLAT_ADDRESS, // Default + llvm::AMDGPUAS::GLOBAL_ADDRESS, // opencl_global + llvm::AMDGPUAS::LOCAL_ADDRESS, // opencl_local + llvm::AMDGPUAS::CONSTANT_ADDRESS, // opencl_constant + llvm::AMDGPUAS::PRIVATE_ADDRESS, // opencl_private + llvm::AMDGPUAS::FLAT_ADDRESS, // opencl_generic + llvm::AMDGPUAS::GLOBAL_ADDRESS, // opencl_global_device + llvm::AMDGPUAS::GLOBAL_ADDRESS, // opencl_global_host + llvm::AMDGPUAS::GLOBAL_ADDRESS, // cuda_device + llvm::AMDGPUAS::CONSTANT_ADDRESS, // cuda_constant + llvm::AMDGPUAS::LOCAL_ADDRESS, // cuda_shared + llvm::AMDGPUAS::GLOBAL_ADDRESS, // sycl_global + llvm::AMDGPUAS::GLOBAL_ADDRESS, // sycl_global_device + llvm::AMDGPUAS::GLOBAL_ADDRESS, // sycl_global_host + llvm::AMDGPUAS::LOCAL_ADDRESS, // sycl_local + llvm::AMDGPUAS::PRIVATE_ADDRESS, // sycl_private + llvm::AMDGPUAS::FLAT_ADDRESS, // ptr32_sptr + llvm::AMDGPUAS::FLAT_ADDRESS, // ptr32_uptr + llvm::AMDGPUAS::FLAT_ADDRESS, // ptr64 + llvm::AMDGPUAS::FLAT_ADDRESS, // hlsl_groupshared }; const LangASMap AMDGPUTargetInfo::AMDGPUDefIsPrivMap = { - Private, // Default - Global, // opencl_global - Local, // opencl_local - Constant, // opencl_constant - Private, // opencl_private - Generic, // opencl_generic - Global, // opencl_global_device - Global, // opencl_global_host - Global, // cuda_device - Constant, // cuda_constant - Local, // cuda_shared + llvm::AMDGPUAS::PRIVATE_ADDRESS, // Default + llvm::AMDGPUAS::GLOBAL_ADDRESS, // opencl_global + llvm::AMDGPUAS::LOCAL_ADDRESS, // opencl_local + llvm::AMDGPUAS::CONSTANT_ADDRESS, // opencl_constant + llvm::AMDGPUAS::PRIVATE_ADDRESS, // opencl_private + llvm::AMDGPUAS::FLAT_ADDRESS, // opencl_generic + llvm::AMDGPUAS::GLOBAL_ADDRESS, // opencl_global_device + llvm::AMDGPUAS::GLOBAL_ADDRESS, // opencl_global_host + llvm::AMDGPUAS::GLOBAL_ADDRESS, // cuda_device + llvm::AMDGPUAS::CONSTANT_ADDRESS, // cuda_constant + llvm::AMDGPUAS::LOCAL_ADDRESS, // cuda_shared // SYCL address space values for this map are dummy - Generic, // sycl_global - Generic, // sycl_global_device - Generic, // sycl_global_host - Generic, // sycl_local - Generic, // sycl_private - Generic, // ptr32_sptr - Generic, // ptr32_uptr - Generic, // ptr64 - Generic, // hlsl_groupshared + llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_global + llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_global_device + llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_global_host + llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_local + llvm::AMDGPUAS::FLAT_ADDRESS, // sycl_private + llvm::AMDGPUAS::FLAT_ADDRESS, // ptr32_sptr + llvm::AMDGPUAS::FLAT_ADDRESS, // ptr32_uptr + llvm::AMDGPUAS::FLAT_ADDRESS, // ptr64 + llvm::AMDGPUAS::FLAT_ADDRESS, // hlsl_groupshared }; } // namespace targets diff --git a/clang/lib/Basic/Targets/AMDGPU.h b/clang/lib/Basic/Targets/AMDGPU.h index 300d9691d8a0..1819ba544ccf 100644 --- a/clang/lib/Basic/Targets/AMDGPU.h +++ b/clang/lib/Basic/Targets/AMDGPU.h @@ -17,6 +17,7 @@ #include "clang/Basic/TargetInfo.h" #include "clang/Basic/TargetOptions.h" #include "llvm/ADT/StringSet.h" +#include "llvm/Support/AMDGPUAddrSpace.h" #include "llvm/Support/Compiler.h" #include "llvm/TargetParser/TargetParser.h" #include "llvm/TargetParser/Triple.h" @@ -29,13 +30,6 @@ class LLVM_LIBRARY_VISIBILITY AMDGPUTargetInfo final : public TargetInfo { static const char *const GCCRegNames[]; - enum AddrSpace { - Generic = 0, - Global = 1, - Local = 3, - Constant = 4, - Private = 5 - }; static const LangASMap AMDGPUDefIsGenMap; static const LangASMap AMDGPUDefIsPrivMap; @@ -106,7 +100,8 @@ public: return 32; unsigned TargetAS = getTargetAddressSpace(AS); - if (TargetAS == Private || TargetAS == Local) + if (TargetAS == llvm::AMDGPUAS::PRIVATE_ADDRESS || + TargetAS == llvm::AMDGPUAS::LOCAL_ADDRESS) return 32; return 64; @@ -376,7 +371,7 @@ public: } std::optional getConstantAddressSpace() const override { - return getLangASFromTargetAS(Constant); + return getLangASFromTargetAS(llvm::AMDGPUAS::CONSTANT_ADDRESS); } const llvm::omp::GV &getGridValue() const override { @@ -392,7 +387,7 @@ public: /// \returns Target specific vtbl ptr address space. unsigned getVtblPtrAddressSpace() const override { - return static_cast(Constant); + return static_cast(llvm::AMDGPUAS::CONSTANT_ADDRESS); } /// \returns If a target requires an address within a target specific address @@ -405,9 +400,9 @@ public: getDWARFAddressSpace(unsigned AddressSpace) const override { const unsigned DWARF_Private = 1; const unsigned DWARF_Local = 2; - if (AddressSpace == Private) { + if (AddressSpace == llvm::AMDGPUAS::PRIVATE_ADDRESS) { return DWARF_Private; - } else if (AddressSpace == Local) { + } else if (AddressSpace == llvm::AMDGPUAS::LOCAL_ADDRESS) { return DWARF_Local; } else { return std::nullopt; diff --git a/clang/lib/Basic/Targets/NVPTX.cpp b/clang/lib/Basic/Targets/NVPTX.cpp index 3a4a75b0348f..5c601812f617 100644 --- a/clang/lib/Basic/Targets/NVPTX.cpp +++ b/clang/lib/Basic/Targets/NVPTX.cpp @@ -262,11 +262,14 @@ void NVPTXTargetInfo::getTargetDefines(const LangOptions &Opts, case CudaArch::SM_89: return "890"; case CudaArch::SM_90: + case CudaArch::SM_90a: return "900"; } llvm_unreachable("unhandled CudaArch"); }(); Builder.defineMacro("__CUDA_ARCH__", CUDAArchCode); + if (GPU == CudaArch::SM_90a) + Builder.defineMacro("__CUDA_ARCH_FEAT_SM90_ALL", "1"); } } diff --git a/clang/lib/Basic/Targets/OSTargets.cpp b/clang/lib/Basic/Targets/OSTargets.cpp index 627bc912fa23..899aefa6173a 100644 --- a/clang/lib/Basic/Targets/OSTargets.cpp +++ b/clang/lib/Basic/Targets/OSTargets.cpp @@ -224,6 +224,9 @@ static void addVisualCDefines(const LangOptions &Opts, MacroBuilder &Builder) { else if (Opts.CPlusPlus14) Builder.defineMacro("_MSVC_LANG", "201402L"); } + + if (Opts.isCompatibleWithMSVC(LangOptions::MSVC2022_3)) + Builder.defineMacro("_MSVC_CONSTEXPR_ATTRIBUTE"); } if (Opts.MicrosoftExt) { diff --git a/clang/lib/Basic/Targets/RISCV.cpp b/clang/lib/Basic/Targets/RISCV.cpp index 13f934e99472..45d23022b530 100644 --- a/clang/lib/Basic/Targets/RISCV.cpp +++ b/clang/lib/Basic/Targets/RISCV.cpp @@ -131,7 +131,7 @@ static unsigned getVersionValue(unsigned MajorVersion, unsigned MinorVersion) { void RISCVTargetInfo::getTargetDefines(const LangOptions &Opts, MacroBuilder &Builder) const { Builder.defineMacro("__riscv"); - bool Is64Bit = getTriple().getArch() == llvm::Triple::riscv64; + bool Is64Bit = getTriple().isRISCV64(); Builder.defineMacro("__riscv_xlen", Is64Bit ? "64" : "32"); StringRef CodeModel = getTargetOpts().CodeModel; unsigned FLen = ISAInfo->getFLen(); @@ -281,7 +281,7 @@ bool RISCVTargetInfo::initFeatureMap( unsigned XLen = 32; - if (getTriple().getArch() == llvm::Triple::riscv64) { + if (getTriple().isRISCV64()) { Features["64bit"] = true; XLen = 64; } else { @@ -304,11 +304,18 @@ bool RISCVTargetInfo::initFeatureMap( // RISCVISAInfo makes implications for ISA features std::vector ImpliedFeatures = (*ParseResult)->toFeatureVector(); - // Add non-ISA features like `relax` and `save-restore` back - for (const std::string &Feature : NewFeaturesVec) - if (!llvm::is_contained(ImpliedFeatures, Feature)) - ImpliedFeatures.push_back(Feature); + // parseFeatures normalizes the feature set by dropping any explicit + // negatives, and non-extension features. We need to preserve the later + // for correctness and want to preserve the former for consistency. + for (auto &Feature : NewFeaturesVec) { + StringRef ExtName = Feature; + assert(ExtName.size() > 1 && (ExtName[0] == '+' || ExtName[0] == '-')); + ExtName = ExtName.drop_front(1); // Drop '+' or '-' + if (!llvm::is_contained(ImpliedFeatures, ("+" + ExtName).str()) && + !llvm::is_contained(ImpliedFeatures, ("-" + ExtName).str())) + ImpliedFeatures.push_back(Feature); + } return TargetInfo::initFeatureMap(Features, Diags, CPU, ImpliedFeatures); } @@ -336,7 +343,7 @@ RISCVTargetInfo::getVScaleRange(const LangOptions &LangOpts) const { /// Return true if has this feature, need to sync with handleTargetFeatures. bool RISCVTargetInfo::hasFeature(StringRef Feature) const { - bool Is64Bit = getTriple().getArch() == llvm::Triple::riscv64; + bool Is64Bit = getTriple().isRISCV64(); auto Result = llvm::StringSwitch>(Feature) .Case("riscv", true) .Case("riscv32", !Is64Bit) @@ -347,10 +354,7 @@ bool RISCVTargetInfo::hasFeature(StringRef Feature) const { if (Result) return *Result; - if (ISAInfo->isSupportedExtensionFeature(Feature)) - return ISAInfo->hasExtension(Feature); - - return false; + return ISAInfo->hasExtension(Feature); } /// Perform initialization based on the user configured set of features. diff --git a/clang/lib/CodeGen/BackendUtil.cpp b/clang/lib/CodeGen/BackendUtil.cpp index 8c666e2cb463..77455c075cab 100644 --- a/clang/lib/CodeGen/BackendUtil.cpp +++ b/clang/lib/CodeGen/BackendUtil.cpp @@ -982,7 +982,7 @@ void EmitAssemblyHelper::RunOptimizationPipeline( getInstrProfOptions(CodeGenOpts, LangOpts)) PB.registerPipelineStartEPCallback( [Options](ModulePassManager &MPM, OptimizationLevel Level) { - MPM.addPass(InstrProfiling(*Options, false)); + MPM.addPass(InstrProfilingLoweringPass(*Options, false)); }); // TODO: Consider passing the MemoryProfileOutput to the pass builder via diff --git a/clang/lib/CodeGen/CGAtomic.cpp b/clang/lib/CodeGen/CGAtomic.cpp index 379c833af32a..52e6ddb7d6af 100644 --- a/clang/lib/CodeGen/CGAtomic.cpp +++ b/clang/lib/CodeGen/CGAtomic.cpp @@ -507,9 +507,11 @@ static llvm::Value *EmitPostAtomicMinMax(CGBuilderTy &Builder, default: llvm_unreachable("Unexpected min/max operation"); case AtomicExpr::AO__atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: Pred = IsSigned ? llvm::CmpInst::ICMP_SGT : llvm::CmpInst::ICMP_UGT; break; case AtomicExpr::AO__atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: Pred = IsSigned ? llvm::CmpInst::ICMP_SLT : llvm::CmpInst::ICMP_ULT; break; } @@ -544,7 +546,9 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, FailureOrder, Size, Order, Scope); return; case AtomicExpr::AO__atomic_compare_exchange: - case AtomicExpr::AO__atomic_compare_exchange_n: { + case AtomicExpr::AO__atomic_compare_exchange_n: + case AtomicExpr::AO__scoped_atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: { if (llvm::ConstantInt *IsWeakC = dyn_cast(IsWeak)) { emitAtomicCmpXchgFailureSet(CGF, E, IsWeakC->getZExtValue(), Dest, Ptr, Val1, Val2, FailureOrder, Size, Order, Scope); @@ -577,7 +581,9 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_load: case AtomicExpr::AO__hip_atomic_load: case AtomicExpr::AO__atomic_load_n: - case AtomicExpr::AO__atomic_load: { + case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__scoped_atomic_load_n: + case AtomicExpr::AO__scoped_atomic_load: { llvm::LoadInst *Load = CGF.Builder.CreateLoad(Ptr); Load->setAtomic(Order, Scope); Load->setVolatile(E->isVolatile()); @@ -589,7 +595,9 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_store: case AtomicExpr::AO__hip_atomic_store: case AtomicExpr::AO__atomic_store: - case AtomicExpr::AO__atomic_store_n: { + case AtomicExpr::AO__atomic_store_n: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_store_n: { llvm::Value *LoadVal1 = CGF.Builder.CreateLoad(Val1); llvm::StoreInst *Store = CGF.Builder.CreateStore(LoadVal1, Ptr); Store->setAtomic(Order, Scope); @@ -602,10 +610,13 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__opencl_atomic_exchange: case AtomicExpr::AO__atomic_exchange_n: case AtomicExpr::AO__atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange_n: + case AtomicExpr::AO__scoped_atomic_exchange: Op = llvm::AtomicRMWInst::Xchg; break; case AtomicExpr::AO__atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_add_fetch: PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FAdd : llvm::Instruction::Add; [[fallthrough]]; @@ -613,11 +624,13 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__hip_atomic_fetch_add: case AtomicExpr::AO__opencl_atomic_fetch_add: case AtomicExpr::AO__atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_add: Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FAdd : llvm::AtomicRMWInst::Add; break; case AtomicExpr::AO__atomic_sub_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: PostOp = E->getValueType()->isFloatingType() ? llvm::Instruction::FSub : llvm::Instruction::Sub; [[fallthrough]]; @@ -625,17 +638,20 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, case AtomicExpr::AO__hip_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_sub: case AtomicExpr::AO__atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_fetch_sub: Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FSub : llvm::AtomicRMWInst::Sub; break; case AtomicExpr::AO__atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: PostOpMinMax = true; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_min: case AtomicExpr::AO__hip_atomic_fetch_min: case AtomicExpr::AO__opencl_atomic_fetch_min: case AtomicExpr::AO__atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_min: Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FMin : (E->getValueType()->isSignedIntegerType() @@ -644,12 +660,14 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, break; case AtomicExpr::AO__atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: PostOpMinMax = true; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_max: case AtomicExpr::AO__hip_atomic_fetch_max: case AtomicExpr::AO__opencl_atomic_fetch_max: case AtomicExpr::AO__atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_max: Op = E->getValueType()->isFloatingType() ? llvm::AtomicRMWInst::FMax : (E->getValueType()->isSignedIntegerType() @@ -658,40 +676,48 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, break; case AtomicExpr::AO__atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_and_fetch: PostOp = llvm::Instruction::And; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_and: case AtomicExpr::AO__hip_atomic_fetch_and: case AtomicExpr::AO__opencl_atomic_fetch_and: case AtomicExpr::AO__atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_and: Op = llvm::AtomicRMWInst::And; break; case AtomicExpr::AO__atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: PostOp = llvm::Instruction::Or; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_or: case AtomicExpr::AO__hip_atomic_fetch_or: case AtomicExpr::AO__opencl_atomic_fetch_or: case AtomicExpr::AO__atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_or: Op = llvm::AtomicRMWInst::Or; break; case AtomicExpr::AO__atomic_xor_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: PostOp = llvm::Instruction::Xor; [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_xor: case AtomicExpr::AO__hip_atomic_fetch_xor: case AtomicExpr::AO__opencl_atomic_fetch_xor: case AtomicExpr::AO__atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_fetch_xor: Op = llvm::AtomicRMWInst::Xor; break; case AtomicExpr::AO__atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: PostOp = llvm::Instruction::And; // the NOT is special cased below [[fallthrough]]; case AtomicExpr::AO__c11_atomic_fetch_nand: case AtomicExpr::AO__atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_nand: Op = llvm::AtomicRMWInst::Nand; break; } @@ -711,7 +737,8 @@ static void EmitAtomicOp(CodeGenFunction &CGF, AtomicExpr *E, Address Dest, else if (PostOp) Result = CGF.Builder.CreateBinOp((llvm::Instruction::BinaryOps)PostOp, RMWI, LoadVal1); - if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) + if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch || + E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch) Result = CGF.Builder.CreateNot(Result); CGF.Builder.CreateStore(Result, Dest); } @@ -861,20 +888,24 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { llvm_unreachable("Already handled above with EmitAtomicInit!"); case AtomicExpr::AO__atomic_load_n: + case AtomicExpr::AO__scoped_atomic_load_n: case AtomicExpr::AO__c11_atomic_load: case AtomicExpr::AO__opencl_atomic_load: case AtomicExpr::AO__hip_atomic_load: break; case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__scoped_atomic_load: Dest = EmitPointerWithAlignment(E->getVal1()); break; case AtomicExpr::AO__atomic_store: + case AtomicExpr::AO__scoped_atomic_store: Val1 = EmitPointerWithAlignment(E->getVal1()); break; case AtomicExpr::AO__atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange: Val1 = EmitPointerWithAlignment(E->getVal1()); Dest = EmitPointerWithAlignment(E->getVal2()); break; @@ -887,14 +918,19 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__hip_atomic_compare_exchange_strong: case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: + case AtomicExpr::AO__scoped_atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: Val1 = EmitPointerWithAlignment(E->getVal1()); - if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange) + if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange || + E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange) Val2 = EmitPointerWithAlignment(E->getVal2()); else Val2 = EmitValToTemp(*this, E->getVal2()); OrderFail = EmitScalarExpr(E->getOrderFail()); if (E->getOp() == AtomicExpr::AO__atomic_compare_exchange_n || - E->getOp() == AtomicExpr::AO__atomic_compare_exchange) + E->getOp() == AtomicExpr::AO__atomic_compare_exchange || + E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange_n || + E->getOp() == AtomicExpr::AO__scoped_atomic_compare_exchange) IsWeak = EmitScalarExpr(E->getWeak()); break; @@ -934,6 +970,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__opencl_atomic_fetch_min: case AtomicExpr::AO__hip_atomic_fetch_max: case AtomicExpr::AO__hip_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: ShouldCastToIntPtrTy = !MemTy->isFloatingType(); [[fallthrough]]; @@ -963,6 +1007,16 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__opencl_atomic_fetch_xor: case AtomicExpr::AO__opencl_atomic_store: case AtomicExpr::AO__opencl_atomic_exchange: + case AtomicExpr::AO__scoped_atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: + case AtomicExpr::AO__scoped_atomic_store_n: + case AtomicExpr::AO__scoped_atomic_exchange_n: Val1 = EmitValToTemp(*this, E->getVal1()); break; } @@ -1039,6 +1093,22 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__opencl_atomic_fetch_or: case AtomicExpr::AO__opencl_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: // For these, only library calls for certain sizes exist. UseOptimizedLibcall = true; break; @@ -1047,6 +1117,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__atomic_store: case AtomicExpr::AO__atomic_exchange: case AtomicExpr::AO__atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_load: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange: // Use the generic version if we don't know that the operand will be // suitably aligned for the optimized version. if (Misaligned) @@ -1071,6 +1145,10 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__opencl_atomic_exchange: case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: + case AtomicExpr::AO__scoped_atomic_load_n: + case AtomicExpr::AO__scoped_atomic_store_n: + case AtomicExpr::AO__scoped_atomic_exchange_n: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: // Only use optimized library calls for sizes for which they exist. // FIXME: Size == 16 optimized library functions exist too. if (Size == 1 || Size == 2 || Size == 4 || Size == 8) @@ -1131,6 +1209,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__hip_atomic_compare_exchange_strong: case AtomicExpr::AO__opencl_atomic_compare_exchange_weak: case AtomicExpr::AO__opencl_atomic_compare_exchange_strong: + case AtomicExpr::AO__scoped_atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: LibCallName = "__atomic_compare_exchange"; RetTy = getContext().BoolTy; HaveRetTy = true; @@ -1150,6 +1230,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__c11_atomic_exchange: case AtomicExpr::AO__hip_atomic_exchange: case AtomicExpr::AO__opencl_atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange_n: LibCallName = "__atomic_exchange"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); @@ -1161,6 +1243,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__c11_atomic_store: case AtomicExpr::AO__hip_atomic_store: case AtomicExpr::AO__opencl_atomic_store: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_store_n: LibCallName = "__atomic_store"; RetTy = getContext().VoidTy; HaveRetTy = true; @@ -1174,17 +1258,21 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { case AtomicExpr::AO__c11_atomic_load: case AtomicExpr::AO__hip_atomic_load: case AtomicExpr::AO__opencl_atomic_load: + case AtomicExpr::AO__scoped_atomic_load: + case AtomicExpr::AO__scoped_atomic_load_n: LibCallName = "__atomic_load"; break; // T __atomic_add_fetch_N(T *mem, T val, int order) // T __atomic_fetch_add_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_add_fetch: PostOp = llvm::Instruction::Add; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_add: case AtomicExpr::AO__c11_atomic_fetch_add: case AtomicExpr::AO__hip_atomic_fetch_add: case AtomicExpr::AO__opencl_atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_add: LibCallName = "__atomic_fetch_add"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), LoweredMemTy, E->getExprLoc(), TInfo.Width); @@ -1192,12 +1280,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_and_fetch_N(T *mem, T val, int order) // T __atomic_fetch_and_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_and_fetch: PostOp = llvm::Instruction::And; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_and: case AtomicExpr::AO__c11_atomic_fetch_and: case AtomicExpr::AO__hip_atomic_fetch_and: case AtomicExpr::AO__opencl_atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_and: LibCallName = "__atomic_fetch_and"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); @@ -1205,12 +1295,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_or_fetch_N(T *mem, T val, int order) // T __atomic_fetch_or_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: PostOp = llvm::Instruction::Or; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_or: case AtomicExpr::AO__c11_atomic_fetch_or: case AtomicExpr::AO__hip_atomic_fetch_or: case AtomicExpr::AO__opencl_atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_or: LibCallName = "__atomic_fetch_or"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); @@ -1218,12 +1310,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_sub_fetch_N(T *mem, T val, int order) // T __atomic_fetch_sub_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_sub_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: PostOp = llvm::Instruction::Sub; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_sub: case AtomicExpr::AO__c11_atomic_fetch_sub: case AtomicExpr::AO__hip_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_fetch_sub: LibCallName = "__atomic_fetch_sub"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), LoweredMemTy, E->getExprLoc(), TInfo.Width); @@ -1231,21 +1325,25 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_xor_fetch_N(T *mem, T val, int order) // T __atomic_fetch_xor_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_xor_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: PostOp = llvm::Instruction::Xor; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_xor: case AtomicExpr::AO__c11_atomic_fetch_xor: case AtomicExpr::AO__hip_atomic_fetch_xor: case AtomicExpr::AO__opencl_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_fetch_xor: LibCallName = "__atomic_fetch_xor"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); break; case AtomicExpr::AO__atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: PostOpMinMax = true; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_min: case AtomicExpr::AO__c11_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_fetch_min: case AtomicExpr::AO__hip_atomic_fetch_min: case AtomicExpr::AO__opencl_atomic_fetch_min: LibCallName = E->getValueType()->isSignedIntegerType() @@ -1255,12 +1353,14 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { LoweredMemTy, E->getExprLoc(), TInfo.Width); break; case AtomicExpr::AO__atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_max_fetch: PostOpMinMax = true; [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_max: case AtomicExpr::AO__c11_atomic_fetch_max: case AtomicExpr::AO__hip_atomic_fetch_max: case AtomicExpr::AO__opencl_atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_max: LibCallName = E->getValueType()->isSignedIntegerType() ? "__atomic_fetch_max" : "__atomic_fetch_umax"; @@ -1270,10 +1370,12 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { // T __atomic_nand_fetch_N(T *mem, T val, int order) // T __atomic_fetch_nand_N(T *mem, T val, int order) case AtomicExpr::AO__atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: PostOp = llvm::Instruction::And; // the NOT is special cased below [[fallthrough]]; case AtomicExpr::AO__atomic_fetch_nand: case AtomicExpr::AO__c11_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_fetch_nand: LibCallName = "__atomic_fetch_nand"; AddDirectArgument(*this, Args, UseOptimizedLibcall, Val1.getPointer(), MemTy, E->getExprLoc(), TInfo.Width); @@ -1330,7 +1432,8 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { llvm::Value *LoadVal1 = Args[1].getRValue(*this).getScalarVal(); ResVal = Builder.CreateBinOp(PostOp, ResVal, LoadVal1); } - if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch) + if (E->getOp() == AtomicExpr::AO__atomic_nand_fetch || + E->getOp() == AtomicExpr::AO__scoped_atomic_nand_fetch) ResVal = Builder.CreateNot(ResVal); Builder.CreateStore(ResVal, Dest.withElementType(ResVal->getType())); @@ -1347,12 +1450,16 @@ RValue CodeGenFunction::EmitAtomicExpr(AtomicExpr *E) { E->getOp() == AtomicExpr::AO__opencl_atomic_store || E->getOp() == AtomicExpr::AO__hip_atomic_store || E->getOp() == AtomicExpr::AO__atomic_store || - E->getOp() == AtomicExpr::AO__atomic_store_n; + E->getOp() == AtomicExpr::AO__atomic_store_n || + E->getOp() == AtomicExpr::AO__scoped_atomic_store || + E->getOp() == AtomicExpr::AO__scoped_atomic_store_n; bool IsLoad = E->getOp() == AtomicExpr::AO__c11_atomic_load || E->getOp() == AtomicExpr::AO__opencl_atomic_load || E->getOp() == AtomicExpr::AO__hip_atomic_load || E->getOp() == AtomicExpr::AO__atomic_load || - E->getOp() == AtomicExpr::AO__atomic_load_n; + E->getOp() == AtomicExpr::AO__atomic_load_n || + E->getOp() == AtomicExpr::AO__scoped_atomic_load || + E->getOp() == AtomicExpr::AO__scoped_atomic_load_n; if (isa(Order)) { auto ord = cast(Order)->getZExtValue(); diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp index 0d8b3e4aaad4..83d0a72aac54 100644 --- a/clang/lib/CodeGen/CGBuiltin.cpp +++ b/clang/lib/CodeGen/CGBuiltin.cpp @@ -10211,6 +10211,22 @@ Value *CodeGenFunction::EmitAArch64SVEBuiltinExpr(unsigned BuiltinID, switch (BuiltinID) { default: return nullptr; + + case SVE::BI__builtin_sve_svreinterpret_b: { + auto SVCountTy = + llvm::TargetExtType::get(getLLVMContext(), "aarch64.svcount"); + Function *CastFromSVCountF = + CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_to_svbool, SVCountTy); + return Builder.CreateCall(CastFromSVCountF, Ops[0]); + } + case SVE::BI__builtin_sve_svreinterpret_c: { + auto SVCountTy = + llvm::TargetExtType::get(getLLVMContext(), "aarch64.svcount"); + Function *CastToSVCountF = + CGM.getIntrinsic(Intrinsic::aarch64_sve_convert_from_svbool, SVCountTy); + return Builder.CreateCall(CastToSVCountF, Ops[0]); + } + case SVE::BI__builtin_sve_svpsel_lane_b8: case SVE::BI__builtin_sve_svpsel_lane_b16: case SVE::BI__builtin_sve_svpsel_lane_b32: diff --git a/clang/lib/CodeGen/CGCUDANV.cpp b/clang/lib/CodeGen/CGCUDANV.cpp index 66147f656071..520b0c4f1176 100644 --- a/clang/lib/CodeGen/CGCUDANV.cpp +++ b/clang/lib/CodeGen/CGCUDANV.cpp @@ -1132,26 +1132,39 @@ void CGNVCUDARuntime::createOffloadingEntries() { for (KernelInfo &I : EmittedKernels) llvm::offloading::emitOffloadingEntry( M, KernelHandles[I.Kernel->getName()], - getDeviceSideName(cast(I.D)), 0, - DeviceVarFlags::OffloadGlobalEntry, Section); + getDeviceSideName(cast(I.D)), /*Flags=*/0, /*Data=*/0, + llvm::offloading::OffloadGlobalEntry, Section); for (VarInfo &I : DeviceVars) { uint64_t VarSize = CGM.getDataLayout().getTypeAllocSize(I.Var->getValueType()); + int32_t Flags = + (I.Flags.isExtern() + ? static_cast(llvm::offloading::OffloadGlobalExtern) + : 0) | + (I.Flags.isConstant() + ? static_cast(llvm::offloading::OffloadGlobalConstant) + : 0) | + (I.Flags.isNormalized() + ? static_cast(llvm::offloading::OffloadGlobalNormalized) + : 0); if (I.Flags.getKind() == DeviceVarFlags::Variable) { llvm::offloading::emitOffloadingEntry( M, I.Var, getDeviceSideName(I.D), VarSize, - I.Flags.isManaged() ? DeviceVarFlags::OffloadGlobalManagedEntry - : DeviceVarFlags::OffloadGlobalEntry, - Section); + (I.Flags.isManaged() ? llvm::offloading::OffloadGlobalManagedEntry + : llvm::offloading::OffloadGlobalEntry) | + Flags, + /*Data=*/0, Section); } else if (I.Flags.getKind() == DeviceVarFlags::Surface) { llvm::offloading::emitOffloadingEntry( M, I.Var, getDeviceSideName(I.D), VarSize, - DeviceVarFlags::OffloadGlobalSurfaceEntry, Section); + llvm::offloading::OffloadGlobalSurfaceEntry | Flags, + I.Flags.getSurfTexType(), Section); } else if (I.Flags.getKind() == DeviceVarFlags::Texture) { llvm::offloading::emitOffloadingEntry( M, I.Var, getDeviceSideName(I.D), VarSize, - DeviceVarFlags::OffloadGlobalTextureEntry, Section); + llvm::offloading::OffloadGlobalTextureEntry | Flags, + I.Flags.getSurfTexType(), Section); } } } diff --git a/clang/lib/CodeGen/CGCUDARuntime.h b/clang/lib/CodeGen/CGCUDARuntime.h index 9a9c6d26cc63..c7af8f1cf0fe 100644 --- a/clang/lib/CodeGen/CGCUDARuntime.h +++ b/clang/lib/CodeGen/CGCUDARuntime.h @@ -17,6 +17,7 @@ #include "clang/AST/GlobalDecl.h" #include "llvm/ADT/StringRef.h" +#include "llvm/Frontend/Offloading/Utility.h" #include "llvm/IR/GlobalValue.h" namespace llvm { @@ -52,19 +53,6 @@ public: Texture, // Builtin texture }; - /// The kind flag for an offloading entry. - enum OffloadEntryKindFlag : uint32_t { - /// Mark the entry as a global entry. This indicates the presense of a - /// kernel if the size field is zero and a variable otherwise. - OffloadGlobalEntry = 0x0, - /// Mark the entry as a managed global variable. - OffloadGlobalManagedEntry = 0x1, - /// Mark the entry as a surface variable. - OffloadGlobalSurfaceEntry = 0x2, - /// Mark the entry as a texture variable. - OffloadGlobalTextureEntry = 0x3, - }; - private: unsigned Kind : 2; unsigned Extern : 1; diff --git a/clang/lib/CodeGen/CGCall.h b/clang/lib/CodeGen/CGCall.h index aee86a3242fd..1c0d15dc932a 100644 --- a/clang/lib/CodeGen/CGCall.h +++ b/clang/lib/CodeGen/CGCall.h @@ -20,6 +20,7 @@ #include "clang/AST/CanonicalType.h" #include "clang/AST/GlobalDecl.h" #include "clang/AST/Type.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/IR/Value.h" namespace llvm { @@ -406,15 +407,13 @@ enum class FnInfoOpts { }; inline FnInfoOpts operator|(FnInfoOpts A, FnInfoOpts B) { - return static_cast( - static_cast>(A) | - static_cast>(B)); + return static_cast(llvm::to_underlying(A) | + llvm::to_underlying(B)); } inline FnInfoOpts operator&(FnInfoOpts A, FnInfoOpts B) { - return static_cast( - static_cast>(A) & - static_cast>(B)); + return static_cast(llvm::to_underlying(A) & + llvm::to_underlying(B)); } inline FnInfoOpts operator|=(FnInfoOpts A, FnInfoOpts B) { diff --git a/clang/lib/CodeGen/CGExprComplex.cpp b/clang/lib/CodeGen/CGExprComplex.cpp index f3cbd1d0451e..e532794b71bd 100644 --- a/clang/lib/CodeGen/CGExprComplex.cpp +++ b/clang/lib/CodeGen/CGExprComplex.cpp @@ -275,6 +275,10 @@ public: ComplexPairTy EmitBinSub(const BinOpInfo &Op); ComplexPairTy EmitBinMul(const BinOpInfo &Op); ComplexPairTy EmitBinDiv(const BinOpInfo &Op); + ComplexPairTy EmitAlgebraicDiv(llvm::Value *A, llvm::Value *B, llvm::Value *C, + llvm::Value *D); + ComplexPairTy EmitRangeReductionDiv(llvm::Value *A, llvm::Value *B, + llvm::Value *C, llvm::Value *D); ComplexPairTy EmitComplexBinOpLibCall(StringRef LibCallName, const BinOpInfo &Op); @@ -781,6 +785,10 @@ ComplexPairTy ComplexExprEmitter::EmitBinMul(const BinOpInfo &Op) { ResR = Builder.CreateFSub(AC, BD, "mul_r"); ResI = Builder.CreateFAdd(AD, BC, "mul_i"); + if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Limited || + Op.FPFeatures.getComplexRange() == LangOptions::CX_Fortran) + return ComplexPairTy(ResR, ResI); + // Emit the test for the real part becoming NaN and create a branch to // handle it. We test for NaN by comparing the number to itself. Value *IsRNaN = Builder.CreateFCmpUNO(ResR, ResR, "isnan_cmp"); @@ -846,23 +854,139 @@ ComplexPairTy ComplexExprEmitter::EmitBinMul(const BinOpInfo &Op) { return ComplexPairTy(ResR, ResI); } +ComplexPairTy ComplexExprEmitter::EmitAlgebraicDiv(llvm::Value *LHSr, + llvm::Value *LHSi, + llvm::Value *RHSr, + llvm::Value *RHSi) { + // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd)) + llvm::Value *DSTr, *DSTi; + + llvm::Value *AC = Builder.CreateFMul(LHSr, RHSr); // a*c + llvm::Value *BD = Builder.CreateFMul(LHSi, RHSi); // b*d + llvm::Value *ACpBD = Builder.CreateFAdd(AC, BD); // ac+bd + + llvm::Value *CC = Builder.CreateFMul(RHSr, RHSr); // c*c + llvm::Value *DD = Builder.CreateFMul(RHSi, RHSi); // d*d + llvm::Value *CCpDD = Builder.CreateFAdd(CC, DD); // cc+dd + + llvm::Value *BC = Builder.CreateFMul(LHSi, RHSr); // b*c + llvm::Value *AD = Builder.CreateFMul(LHSr, RHSi); // a*d + llvm::Value *BCmAD = Builder.CreateFSub(BC, AD); // bc-ad + + DSTr = Builder.CreateFDiv(ACpBD, CCpDD); + DSTi = Builder.CreateFDiv(BCmAD, CCpDD); + return ComplexPairTy(DSTr, DSTi); +} + +// EmitFAbs - Emit a call to @llvm.fabs. +static llvm::Value *EmitllvmFAbs(CodeGenFunction &CGF, llvm::Value *Value) { + llvm::Function *Func = + CGF.CGM.getIntrinsic(llvm::Intrinsic::fabs, Value->getType()); + llvm::Value *Call = CGF.Builder.CreateCall(Func, Value); + return Call; +} + +// EmitRangeReductionDiv - Implements Smith's algorithm for complex division. +// SMITH, R. L. Algorithm 116: Complex division. Commun. ACM 5, 8 (1962). +ComplexPairTy ComplexExprEmitter::EmitRangeReductionDiv(llvm::Value *LHSr, + llvm::Value *LHSi, + llvm::Value *RHSr, + llvm::Value *RHSi) { + // (a + ib) / (c + id) = (e + if) + llvm::Value *FAbsRHSr = EmitllvmFAbs(CGF, RHSr); // |c| + llvm::Value *FAbsRHSi = EmitllvmFAbs(CGF, RHSi); // |d| + // |c| >= |d| + llvm::Value *IsR = Builder.CreateFCmpUGT(FAbsRHSr, FAbsRHSi, "abs_cmp"); + + llvm::BasicBlock *TrueBB = + CGF.createBasicBlock("abs_rhsr_greater_or_equal_abs_rhsi"); + llvm::BasicBlock *FalseBB = + CGF.createBasicBlock("abs_rhsr_less_than_abs_rhsi"); + llvm::BasicBlock *ContBB = CGF.createBasicBlock("complex_div"); + Builder.CreateCondBr(IsR, TrueBB, FalseBB); + + CGF.EmitBlock(TrueBB); + // abs(c) >= abs(d) + // r = d/c + // tmp = c + rd + // e = (a + br)/tmp + // f = (b - ar)/tmp + llvm::Value *DdC = Builder.CreateFDiv(RHSi, RHSr); // r=d/c + + llvm::Value *RD = Builder.CreateFMul(DdC, RHSi); // rd + llvm::Value *CpRD = Builder.CreateFAdd(RHSr, RD); // tmp=c+rd + + llvm::Value *T3 = Builder.CreateFMul(LHSi, DdC); // br + llvm::Value *T4 = Builder.CreateFAdd(LHSr, T3); // a+br + llvm::Value *DSTTr = Builder.CreateFDiv(T4, CpRD); // (a+br)/tmp + + llvm::Value *T5 = Builder.CreateFMul(LHSr, DdC); // ar + llvm::Value *T6 = Builder.CreateFSub(LHSi, T5); // b-ar + llvm::Value *DSTTi = Builder.CreateFDiv(T6, CpRD); // (b-ar)/tmp + Builder.CreateBr(ContBB); + + CGF.EmitBlock(FalseBB); + // abs(c) < abs(d) + // r = c/d + // tmp = d + rc + // e = (ar + b)/tmp + // f = (br - a)/tmp + llvm::Value *CdD = Builder.CreateFDiv(RHSr, RHSi); // r=c/d + + llvm::Value *RC = Builder.CreateFMul(CdD, RHSr); // rc + llvm::Value *DpRC = Builder.CreateFAdd(RHSi, RC); // tmp=d+rc + + llvm::Value *T7 = Builder.CreateFMul(LHSr, RC); // ar + llvm::Value *T8 = Builder.CreateFAdd(T7, LHSi); // ar+b + llvm::Value *DSTFr = Builder.CreateFDiv(T8, DpRC); // (ar+b)/tmp + + llvm::Value *T9 = Builder.CreateFMul(LHSi, CdD); // br + llvm::Value *T10 = Builder.CreateFSub(T9, LHSr); // br-a + llvm::Value *DSTFi = Builder.CreateFDiv(T10, DpRC); // (br-a)/tmp + Builder.CreateBr(ContBB); + + // Phi together the computation paths. + CGF.EmitBlock(ContBB); + llvm::PHINode *VALr = Builder.CreatePHI(DSTTr->getType(), 2); + VALr->addIncoming(DSTTr, TrueBB); + VALr->addIncoming(DSTFr, FalseBB); + llvm::PHINode *VALi = Builder.CreatePHI(DSTTi->getType(), 2); + VALi->addIncoming(DSTTi, TrueBB); + VALi->addIncoming(DSTFi, FalseBB); + return ComplexPairTy(VALr, VALi); +} + // See C11 Annex G.5.1 for the semantics of multiplicative operators on complex // typed values. ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) { llvm::Value *LHSr = Op.LHS.first, *LHSi = Op.LHS.second; llvm::Value *RHSr = Op.RHS.first, *RHSi = Op.RHS.second; - llvm::Value *DSTr, *DSTi; if (LHSr->getType()->isFloatingPointTy()) { - // If we have a complex operand on the RHS and FastMath is not allowed, we - // delegate to a libcall to handle all of the complexities and minimize - // underflow/overflow cases. When FastMath is allowed we construct the - // divide inline using the same algorithm as for integer operands. - // - // FIXME: We would be able to avoid the libcall in many places if we - // supported imaginary types in addition to complex types. CodeGenFunction::CGFPOptionsRAII FPOptsRAII(CGF, Op.FPFeatures); - if (RHSi && !CGF.getLangOpts().FastMath) { + if (!RHSi) { + assert(LHSi && "Can have at most one non-complex operand!"); + + DSTr = Builder.CreateFDiv(LHSr, RHSr); + DSTi = Builder.CreateFDiv(LHSi, RHSr); + return ComplexPairTy(DSTr, DSTi); + } + llvm::Value *OrigLHSi = LHSi; + if (!LHSi) + LHSi = llvm::Constant::getNullValue(RHSi->getType()); + if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Fortran) + return EmitRangeReductionDiv(LHSr, LHSi, RHSr, RHSi); + else if (Op.FPFeatures.getComplexRange() == LangOptions::CX_Limited) + return EmitAlgebraicDiv(LHSr, LHSi, RHSr, RHSi); + else if (!CGF.getLangOpts().FastMath) { + LHSi = OrigLHSi; + // If we have a complex operand on the RHS and FastMath is not allowed, we + // delegate to a libcall to handle all of the complexities and minimize + // underflow/overflow cases. When FastMath is allowed we construct the + // divide inline using the same algorithm as for integer operands. + // + // FIXME: We would be able to avoid the libcall in many places if we + // supported imaginary types in addition to complex types. BinOpInfo LibCallOp = Op; // If LHS was a real, supply a null imaginary part. if (!LHSi) @@ -884,30 +1008,8 @@ ComplexPairTy ComplexExprEmitter::EmitBinDiv(const BinOpInfo &Op) { case llvm::Type::FP128TyID: return EmitComplexBinOpLibCall("__divtc3", LibCallOp); } - } else if (RHSi) { - if (!LHSi) - LHSi = llvm::Constant::getNullValue(RHSi->getType()); - - // (a+ib) / (c+id) = ((ac+bd)/(cc+dd)) + i((bc-ad)/(cc+dd)) - llvm::Value *AC = Builder.CreateFMul(LHSr, RHSr); // a*c - llvm::Value *BD = Builder.CreateFMul(LHSi, RHSi); // b*d - llvm::Value *ACpBD = Builder.CreateFAdd(AC, BD); // ac+bd - - llvm::Value *CC = Builder.CreateFMul(RHSr, RHSr); // c*c - llvm::Value *DD = Builder.CreateFMul(RHSi, RHSi); // d*d - llvm::Value *CCpDD = Builder.CreateFAdd(CC, DD); // cc+dd - - llvm::Value *BC = Builder.CreateFMul(LHSi, RHSr); // b*c - llvm::Value *AD = Builder.CreateFMul(LHSr, RHSi); // a*d - llvm::Value *BCmAD = Builder.CreateFSub(BC, AD); // bc-ad - - DSTr = Builder.CreateFDiv(ACpBD, CCpDD); - DSTi = Builder.CreateFDiv(BCmAD, CCpDD); } else { - assert(LHSi && "Can have at most one non-complex operand!"); - - DSTr = Builder.CreateFDiv(LHSr, RHSr); - DSTi = Builder.CreateFDiv(LHSi, RHSr); + return EmitAlgebraicDiv(LHSr, LHSi, RHSr, RHSi); } } else { assert(Op.LHS.second && Op.RHS.second && diff --git a/clang/lib/CodeGen/CGHLSLRuntime.cpp b/clang/lib/CodeGen/CGHLSLRuntime.cpp index c239bc17ef26..3e8a40e7540b 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.cpp +++ b/clang/lib/CodeGen/CGHLSLRuntime.cpp @@ -184,7 +184,8 @@ void CGHLSLRuntime::finishCodeGen() { : llvm::hlsl::ResourceKind::TBuffer; std::string TyName = Buf.Name.str() + (Buf.IsCBuffer ? ".cb." : ".tb.") + "ty"; - addBufferResourceAnnotation(GV, TyName, RC, RK, Buf.Binding); + addBufferResourceAnnotation(GV, TyName, RC, RK, /*IsROV=*/false, + Buf.Binding); } } @@ -196,6 +197,7 @@ void CGHLSLRuntime::addBufferResourceAnnotation(llvm::GlobalVariable *GV, llvm::StringRef TyName, llvm::hlsl::ResourceClass RC, llvm::hlsl::ResourceKind RK, + bool IsROV, BufferResBinding &Binding) { llvm::Module &M = CGM.getModule(); @@ -219,7 +221,7 @@ void CGHLSLRuntime::addBufferResourceAnnotation(llvm::GlobalVariable *GV, "ResourceMD must have been set by the switch above."); llvm::hlsl::FrontendResource Res( - GV, TyName, RK, Binding.Reg.value_or(UINT_MAX), Binding.Space); + GV, TyName, RK, IsROV, Binding.Reg.value_or(UINT_MAX), Binding.Space); ResourceMD->addOperand(Res.getMetadata()); } @@ -236,10 +238,11 @@ void CGHLSLRuntime::annotateHLSLResource(const VarDecl *D, GlobalVariable *GV) { llvm::hlsl::ResourceClass RC = Attr->getResourceClass(); llvm::hlsl::ResourceKind RK = Attr->getResourceKind(); + bool IsROV = Attr->getIsROV(); QualType QT(Ty, 0); BufferResBinding Binding(D->getAttr()); - addBufferResourceAnnotation(GV, QT.getAsString(), RC, RK, Binding); + addBufferResourceAnnotation(GV, QT.getAsString(), RC, RK, IsROV, Binding); } CGHLSLRuntime::BufferResBinding::BufferResBinding( diff --git a/clang/lib/CodeGen/CGHLSLRuntime.h b/clang/lib/CodeGen/CGHLSLRuntime.h index 67413fbd4a78..bb500cb5c979 100644 --- a/clang/lib/CodeGen/CGHLSLRuntime.h +++ b/clang/lib/CodeGen/CGHLSLRuntime.h @@ -92,7 +92,7 @@ private: void addBufferResourceAnnotation(llvm::GlobalVariable *GV, llvm::StringRef TyName, llvm::hlsl::ResourceClass RC, - llvm::hlsl::ResourceKind RK, + llvm::hlsl::ResourceKind RK, bool IsROV, BufferResBinding &Binding); void addConstant(VarDecl *D, Buffer &CB); void addBufferDecls(const DeclContext *DC, Buffer &CB); diff --git a/clang/lib/CodeGen/CGOpenMPRuntime.cpp b/clang/lib/CodeGen/CGOpenMPRuntime.cpp index 55648963df36..7f7e6f530666 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntime.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntime.cpp @@ -7389,7 +7389,14 @@ private: } else if (FieldIndex < PartialStruct.LowestElem.first) { PartialStruct.LowestElem = {FieldIndex, LowestElem}; } else if (FieldIndex > PartialStruct.HighestElem.first) { - PartialStruct.HighestElem = {FieldIndex, LowestElem}; + if (IsFinalArraySection) { + Address HB = + CGF.EmitOMPArraySectionExpr(OASE, /*IsLowerBound=*/false) + .getAddress(CGF); + PartialStruct.HighestElem = {FieldIndex, HB}; + } else { + PartialStruct.HighestElem = {FieldIndex, LowestElem}; + } } } diff --git a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp index 293ccaa3413c..299ee1460b3d 100644 --- a/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp +++ b/clang/lib/CodeGen/CGOpenMPRuntimeGPU.cpp @@ -3483,6 +3483,7 @@ void CGOpenMPRuntimeGPU::processRequiresDirective( case CudaArch::SM_87: case CudaArch::SM_89: case CudaArch::SM_90: + case CudaArch::SM_90a: case CudaArch::GFX600: case CudaArch::GFX601: case CudaArch::GFX602: diff --git a/clang/lib/CodeGen/CodeGenModule.cpp b/clang/lib/CodeGen/CodeGenModule.cpp index 6a20723bf2bc..b931a81bc008 100644 --- a/clang/lib/CodeGen/CodeGenModule.cpp +++ b/clang/lib/CodeGen/CodeGenModule.cpp @@ -6439,7 +6439,7 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( VD, E->getManglingNumber(), Out); APValue *Value = nullptr; - if (E->getStorageDuration() == SD_Static && VD && VD->evaluateValue()) { + if (E->getStorageDuration() == SD_Static && VD->evaluateValue()) { // If the initializer of the extending declaration is a constant // initializer, we should have a cached constant initializer for this // temporary. Note that this might have a different value from the value @@ -6454,8 +6454,7 @@ ConstantAddress CodeGenModule::GetAddrOfGlobalTemporary( !EvalResult.hasSideEffects()) Value = &EvalResult.Val; - LangAS AddrSpace = - VD ? GetGlobalVarAddressSpace(VD) : MaterializedType.getAddressSpace(); + LangAS AddrSpace = GetGlobalVarAddressSpace(VD); std::optional emitter; llvm::Constant *InitialValue = nullptr; diff --git a/clang/lib/CodeGen/CodeGenTBAA.cpp b/clang/lib/CodeGen/CodeGenTBAA.cpp index 5906b14dd93c..dc288bc3f615 100644 --- a/clang/lib/CodeGen/CodeGenTBAA.cpp +++ b/clang/lib/CodeGen/CodeGenTBAA.cpp @@ -196,11 +196,14 @@ llvm::MDNode *CodeGenTBAA::getTypeInfoHelper(const Type *Ty) { // Enum types are distinct types. In C++ they have "underlying types", // however they aren't related for TBAA. if (const EnumType *ETy = dyn_cast(Ty)) { + if (!Features.CPlusPlus) + return getTypeInfo(ETy->getDecl()->getIntegerType()); + // In C++ mode, types have linkage, so we can rely on the ODR and // on their mangled names, if they're external. // TODO: Is there a way to get a program-wide unique name for a // decl with local linkage or no linkage? - if (!Features.CPlusPlus || !ETy->getDecl()->isExternallyVisible()) + if (!ETy->getDecl()->isExternallyVisible()) return getChar(); SmallString<256> OutName; diff --git a/clang/lib/CodeGen/Targets/AMDGPU.cpp b/clang/lib/CodeGen/Targets/AMDGPU.cpp index b654e3f12af8..03ac6b78598f 100644 --- a/clang/lib/CodeGen/Targets/AMDGPU.cpp +++ b/clang/lib/CodeGen/Targets/AMDGPU.cpp @@ -471,20 +471,25 @@ AMDGPUTargetCodeGenInfo::getLLVMSyncScopeID(const LangOptions &LangOpts, std::string Name; switch (Scope) { case SyncScope::HIPSingleThread: + case SyncScope::SingleScope: Name = "singlethread"; break; case SyncScope::HIPWavefront: case SyncScope::OpenCLSubGroup: + case SyncScope::WavefrontScope: Name = "wavefront"; break; case SyncScope::HIPWorkgroup: case SyncScope::OpenCLWorkGroup: + case SyncScope::WorkgroupScope: Name = "workgroup"; break; case SyncScope::HIPAgent: case SyncScope::OpenCLDevice: + case SyncScope::DeviceScope: Name = "agent"; break; + case SyncScope::SystemScope: case SyncScope::HIPSystem: case SyncScope::OpenCLAllSVMDevices: Name = ""; diff --git a/clang/lib/CrossTU/CrossTranslationUnit.cpp b/clang/lib/CrossTU/CrossTranslationUnit.cpp index 540c22d07865..94c10e50d7d0 100644 --- a/clang/lib/CrossTU/CrossTranslationUnit.cpp +++ b/clang/lib/CrossTU/CrossTranslationUnit.cpp @@ -551,7 +551,7 @@ CrossTranslationUnitContext::ASTLoader::load(StringRef Identifier) { // Normalize by removing relative path components. llvm::sys::path::remove_dots(Path, /*remove_dot_dot*/ true, PathStyle); - if (Path.endswith(".ast")) + if (Path.ends_with(".ast")) return loadFromDump(Path); else return loadFromSource(Path); diff --git a/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp b/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp index 9b3d2571f29f..beca9586988b 100644 --- a/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp +++ b/clang/lib/DirectoryWatcher/linux/DirectoryWatcher-linux.cpp @@ -14,7 +14,6 @@ #include "llvm/Support/AlignOf.h" #include "llvm/Support/Errno.h" #include "llvm/Support/Error.h" -#include "llvm/Support/MathExtras.h" #include "llvm/Support/Path.h" #include #include @@ -25,6 +24,7 @@ #include #include +#include #include #include #include diff --git a/clang/lib/Driver/Driver.cpp b/clang/lib/Driver/Driver.cpp index e241706b9082..f392f6794f85 100644 --- a/clang/lib/Driver/Driver.cpp +++ b/clang/lib/Driver/Driver.cpp @@ -1522,7 +1522,7 @@ bool Driver::getCrashDiagnosticFile(StringRef ReproCrashFilename, // (or /Library/Logs/DiagnosticReports for root) and has the filename pattern // clang-__.crash. path::home_directory(CrashDiagDir); - if (CrashDiagDir.startswith("/var/root")) + if (CrashDiagDir.starts_with("/var/root")) CrashDiagDir = "/"; path::append(CrashDiagDir, "Library/Logs/DiagnosticReports"); int PID = diff --git a/clang/lib/Driver/Multilib.cpp b/clang/lib/Driver/Multilib.cpp index 7681c1a3ce67..9c091bbfdaba 100644 --- a/clang/lib/Driver/Multilib.cpp +++ b/clang/lib/Driver/Multilib.cpp @@ -95,7 +95,7 @@ MultilibSet &MultilibSet::FilterOut(FilterCallback F) { void MultilibSet::push_back(const Multilib &M) { Multilibs.push_back(M); } bool MultilibSet::select(const Multilib::flags_list &Flags, - llvm::SmallVector &Selected) const { + llvm::SmallVectorImpl &Selected) const { llvm::StringSet<> FlagSet(expandFlags(Flags)); Selected.clear(); diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp index f02f7c841b91..f95f3227aba7 100644 --- a/clang/lib/Driver/ToolChains/Clang.cpp +++ b/clang/lib/Driver/ToolChains/Clang.cpp @@ -1294,6 +1294,9 @@ void Clang::AddPreprocessingOptions(Compilation &C, const JobAction &JA, CmdArgs.push_back("-source-date-epoch"); CmdArgs.push_back(Args.MakeArgString(Epoch)); } + + Args.addOptInFlag(CmdArgs, options::OPT_fdefine_target_os_macros, + options::OPT_fno_define_target_os_macros); } // FIXME: Move to target hook. @@ -2657,6 +2660,35 @@ static void CollectArgsForIntegratedAssembler(Compilation &C, } } +static StringRef EnumComplexRangeToStr(LangOptions::ComplexRangeKind Range) { + StringRef RangeStr = ""; + switch (Range) { + case LangOptions::ComplexRangeKind::CX_Limited: + return "-fcx-limited-range"; + break; + case LangOptions::ComplexRangeKind::CX_Fortran: + return "-fcx-fortran-rules"; + break; + default: + return RangeStr; + break; + } +} + +static void EmitComplexRangeDiag(const Driver &D, + LangOptions::ComplexRangeKind Range1, + LangOptions::ComplexRangeKind Range2) { + if (Range1 != LangOptions::ComplexRangeKind::CX_Full) + D.Diag(clang::diag::warn_drv_overriding_option) + << EnumComplexRangeToStr(Range1) << EnumComplexRangeToStr(Range2); +} + +static std::string RenderComplexRangeOption(std::string Range) { + std::string ComplexRangeStr = "-complex-range="; + ComplexRangeStr += Range; + return ComplexRangeStr; +} + static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, bool OFastEnabled, const ArgList &Args, ArgStringList &CmdArgs, @@ -2703,6 +2735,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, bool StrictFPModel = false; StringRef Float16ExcessPrecision = ""; StringRef BFloat16ExcessPrecision = ""; + LangOptions::ComplexRangeKind Range = LangOptions::ComplexRangeKind::CX_Full; if (const Arg *A = Args.getLastArg(options::OPT_flimited_precision_EQ)) { CmdArgs.push_back("-mlimit-float-precision"); @@ -2715,6 +2748,28 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, switch (optID) { default: break; + case options::OPT_fcx_limited_range: { + EmitComplexRangeDiag(D, Range, LangOptions::ComplexRangeKind::CX_Limited); + Range = LangOptions::ComplexRangeKind::CX_Limited; + std::string ComplexRangeStr = RenderComplexRangeOption("limited"); + if (!ComplexRangeStr.empty()) + CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr)); + break; + } + case options::OPT_fno_cx_limited_range: + Range = LangOptions::ComplexRangeKind::CX_Full; + break; + case options::OPT_fcx_fortran_rules: { + EmitComplexRangeDiag(D, Range, LangOptions::ComplexRangeKind::CX_Fortran); + Range = LangOptions::ComplexRangeKind::CX_Fortran; + std::string ComplexRangeStr = RenderComplexRangeOption("fortran"); + if (!ComplexRangeStr.empty()) + CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr)); + break; + } + case options::OPT_fno_cx_fortran_rules: + Range = LangOptions::ComplexRangeKind::CX_Full; + break; case options::OPT_ffp_model_EQ: { // If -ffp-model= is seen, reset to fno-fast-math HonorINFs = true; @@ -2769,7 +2824,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, D.Diag(diag::err_drv_unsupported_option_argument) << A->getSpelling() << Val; break; - } + } } switch (optID) { @@ -2968,7 +3023,7 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, if (!OFastEnabled) continue; [[fallthrough]]; - case options::OPT_ffast_math: + case options::OPT_ffast_math: { HonorINFs = false; HonorNaNs = false; MathErrno = false; @@ -2982,7 +3037,13 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, // If fast-math is set then set the fp-contract mode to fast. FPContract = "fast"; SeenUnsafeMathModeOption = true; + // ffast-math enables fortran rules for complex multiplication and + // division. + std::string ComplexRangeStr = RenderComplexRangeOption("limited"); + if (!ComplexRangeStr.empty()) + CmdArgs.push_back(Args.MakeArgString(ComplexRangeStr)); break; + } case options::OPT_fno_fast_math: HonorINFs = true; HonorNaNs = true; @@ -3136,6 +3197,15 @@ static void RenderFloatingPointOptions(const ToolChain &TC, const Driver &D, if (Args.hasFlag(options::OPT_fno_strict_float_cast_overflow, options::OPT_fstrict_float_cast_overflow, false)) CmdArgs.push_back("-fno-strict-float-cast-overflow"); + + if (const Arg *A = Args.getLastArg(options::OPT_fcx_limited_range)) + CmdArgs.push_back("-fcx-limited-range"); + if (const Arg *A = Args.getLastArg(options::OPT_fcx_fortran_rules)) + CmdArgs.push_back("-fcx-fortran-rules"); + if (const Arg *A = Args.getLastArg(options::OPT_fno_cx_limited_range)) + CmdArgs.push_back("-fno-cx-limited-range"); + if (const Arg *A = Args.getLastArg(options::OPT_fno_cx_fortran_rules)) + CmdArgs.push_back("-fno-cx-fortran-rules"); } static void RenderAnalyzerOptions(const ArgList &Args, ArgStringList &CmdArgs, diff --git a/clang/lib/Driver/ToolChains/CommonArgs.cpp b/clang/lib/Driver/ToolChains/CommonArgs.cpp index 0ae8e2dce32e..31e7d68161ff 100644 --- a/clang/lib/Driver/ToolChains/CommonArgs.cpp +++ b/clang/lib/Driver/ToolChains/CommonArgs.cpp @@ -1120,55 +1120,61 @@ void tools::addFortranRuntimeLibs(const ToolChain &TC, const ArgList &Args, llvm::opt::ArgStringList &CmdArgs) { // These are handled earlier on Windows by telling the frontend driver to add // the correct libraries to link against as dependents in the object file. - if (!TC.getTriple().isKnownWindowsMSVCEnvironment()) { - // The --whole-archive option needs to be part of the link line to - // make sure that the main() function from Fortran_main.a is pulled - // in by the linker. Determine if --whole-archive is active when - // flang will try to link Fortran_main.a. If it is, don't add the - // --whole-archive flag to the link line. If it's not, add a proper - // --whole-archive/--no-whole-archive bracket to the link line. - bool WholeArchiveActive = false; - for (auto *Arg : Args.filtered(options::OPT_Wl_COMMA)) - if (Arg) - for (StringRef ArgValue : Arg->getValues()) { - if (ArgValue == "--whole-archive") - WholeArchiveActive = true; - if (ArgValue == "--no-whole-archive") - WholeArchiveActive = false; - } - if (!WholeArchiveActive) - CmdArgs.push_back("--whole-archive"); - CmdArgs.push_back("-lFortran_main"); - if (!WholeArchiveActive) - CmdArgs.push_back("--no-whole-archive"); + // if -fno-fortran-main has been passed, skip linking Fortran_main.a + bool LinkFortranMain = !Args.hasArg(options::OPT_no_fortran_main); + if (!TC.getTriple().isKnownWindowsMSVCEnvironment()) { + if (LinkFortranMain) { + // The --whole-archive option needs to be part of the link line to + // make sure that the main() function from Fortran_main.a is pulled + // in by the linker. Determine if --whole-archive is active when + // flang will try to link Fortran_main.a. If it is, don't add the + // --whole-archive flag to the link line. If it's not, add a proper + // --whole-archive/--no-whole-archive bracket to the link line. + bool WholeArchiveActive = false; + for (auto *Arg : Args.filtered(options::OPT_Wl_COMMA)) + if (Arg) + for (StringRef ArgValue : Arg->getValues()) { + if (ArgValue == "--whole-archive") + WholeArchiveActive = true; + if (ArgValue == "--no-whole-archive") + WholeArchiveActive = false; + } + if (!WholeArchiveActive) + CmdArgs.push_back("--whole-archive"); + CmdArgs.push_back("-lFortran_main"); + if (!WholeArchiveActive) + CmdArgs.push_back("--no-whole-archive"); + } // Perform regular linkage of the remaining runtime libraries. CmdArgs.push_back("-lFortranRuntime"); CmdArgs.push_back("-lFortranDecimal"); } else { - unsigned RTOptionID = options::OPT__SLASH_MT; - if (auto *rtl = Args.getLastArg(options::OPT_fms_runtime_lib_EQ)) { - RTOptionID = llvm::StringSwitch(rtl->getValue()) - .Case("static", options::OPT__SLASH_MT) - .Case("static_dbg", options::OPT__SLASH_MTd) - .Case("dll", options::OPT__SLASH_MD) - .Case("dll_dbg", options::OPT__SLASH_MDd) - .Default(options::OPT__SLASH_MT); - } - switch (RTOptionID) { - case options::OPT__SLASH_MT: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static.lib"); - break; - case options::OPT__SLASH_MTd: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static_dbg.lib"); - break; - case options::OPT__SLASH_MD: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic.lib"); - break; - case options::OPT__SLASH_MDd: - CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic_dbg.lib"); - break; + if (LinkFortranMain) { + unsigned RTOptionID = options::OPT__SLASH_MT; + if (auto *rtl = Args.getLastArg(options::OPT_fms_runtime_lib_EQ)) { + RTOptionID = llvm::StringSwitch(rtl->getValue()) + .Case("static", options::OPT__SLASH_MT) + .Case("static_dbg", options::OPT__SLASH_MTd) + .Case("dll", options::OPT__SLASH_MD) + .Case("dll_dbg", options::OPT__SLASH_MDd) + .Default(options::OPT__SLASH_MT); + } + switch (RTOptionID) { + case options::OPT__SLASH_MT: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static.lib"); + break; + case options::OPT__SLASH_MTd: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.static_dbg.lib"); + break; + case options::OPT__SLASH_MD: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic.lib"); + break; + case options::OPT__SLASH_MDd: + CmdArgs.push_back("/WHOLEARCHIVE:Fortran_main.dynamic_dbg.lib"); + break; + } } } } @@ -2583,9 +2589,7 @@ void tools::addMachineOutlinerArgs(const Driver &D, // We only support -moutline in AArch64 and ARM targets right now. If // we're not compiling for these, emit a warning and ignore the flag. // Otherwise, add the proper mllvm flags. - if (!(Triple.isARM() || Triple.isThumb() || - Triple.getArch() == llvm::Triple::aarch64 || - Triple.getArch() == llvm::Triple::aarch64_32)) { + if (!(Triple.isARM() || Triple.isThumb() || Triple.isAArch64())) { D.Diag(diag::warn_drv_moutline_unsupported_opt) << Triple.getArchName(); } else { addArg(Twine("-enable-machine-outliner")); diff --git a/clang/lib/Driver/ToolChains/Cuda.cpp b/clang/lib/Driver/ToolChains/Cuda.cpp index e95ff98e6c94..ef1e77974c1e 100644 --- a/clang/lib/Driver/ToolChains/Cuda.cpp +++ b/clang/lib/Driver/ToolChains/Cuda.cpp @@ -78,6 +78,10 @@ CudaVersion getCudaVersion(uint32_t raw_version) { return CudaVersion::CUDA_120; if (raw_version < 12020) return CudaVersion::CUDA_121; + if (raw_version < 12030) + return CudaVersion::CUDA_122; + if (raw_version < 12040) + return CudaVersion::CUDA_123; return CudaVersion::NEW; } @@ -671,6 +675,8 @@ void NVPTX::getNVPTXTargetFeatures(const Driver &D, const llvm::Triple &Triple, case CudaVersion::CUDA_##CUDA_VER: \ PtxFeature = "+ptx" #PTX_VER; \ break; + CASE_CUDA_VERSION(123, 83); + CASE_CUDA_VERSION(122, 82); CASE_CUDA_VERSION(121, 81); CASE_CUDA_VERSION(120, 80); CASE_CUDA_VERSION(118, 78); diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp index f09bc27d7d2c..692b3a3f285d 100644 --- a/clang/lib/Driver/ToolChains/Darwin.cpp +++ b/clang/lib/Driver/ToolChains/Darwin.cpp @@ -1281,7 +1281,7 @@ void MachO::AddLinkRuntimeLib(const ArgList &Args, ArgStringList &CmdArgs, // rpaths. This is currently true from this place, but we need to be // careful if this function is ever called before user's rpaths are emitted. if (Opts & RLO_AddRPath) { - assert(DarwinLibName.endswith(".dylib") && "must be a dynamic library"); + assert(DarwinLibName.ends_with(".dylib") && "must be a dynamic library"); // Add @executable_path to rpath to support having the dylib copied with // the executable. diff --git a/clang/lib/Driver/ToolChains/Flang.cpp b/clang/lib/Driver/ToolChains/Flang.cpp index 9b21fe952af7..502b9f17a06c 100644 --- a/clang/lib/Driver/ToolChains/Flang.cpp +++ b/clang/lib/Driver/ToolChains/Flang.cpp @@ -231,6 +231,8 @@ static void processVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args, ArgStringList &CmdArgs) { assert(TC.getTriple().isKnownWindowsMSVCEnvironment() && "can only add VS runtime library on Windows!"); + // if -fno-fortran-main has been passed, skip linking Fortran_main.a + bool LinkFortranMain = !Args.hasArg(options::OPT_no_fortran_main); if (TC.getTriple().isKnownWindowsMSVCEnvironment()) { CmdArgs.push_back(Args.MakeArgString( "--dependent-lib=" + TC.getCompilerRTBasename(Args, "builtins"))); @@ -248,7 +250,8 @@ static void processVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args, case options::OPT__SLASH_MT: CmdArgs.push_back("-D_MT"); CmdArgs.push_back("--dependent-lib=libcmt"); - CmdArgs.push_back("--dependent-lib=Fortran_main.static.lib"); + if (LinkFortranMain) + CmdArgs.push_back("--dependent-lib=Fortran_main.static.lib"); CmdArgs.push_back("--dependent-lib=FortranRuntime.static.lib"); CmdArgs.push_back("--dependent-lib=FortranDecimal.static.lib"); break; @@ -256,7 +259,8 @@ static void processVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args, CmdArgs.push_back("-D_MT"); CmdArgs.push_back("-D_DEBUG"); CmdArgs.push_back("--dependent-lib=libcmtd"); - CmdArgs.push_back("--dependent-lib=Fortran_main.static_dbg.lib"); + if (LinkFortranMain) + CmdArgs.push_back("--dependent-lib=Fortran_main.static_dbg.lib"); CmdArgs.push_back("--dependent-lib=FortranRuntime.static_dbg.lib"); CmdArgs.push_back("--dependent-lib=FortranDecimal.static_dbg.lib"); break; @@ -264,7 +268,8 @@ static void processVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args, CmdArgs.push_back("-D_MT"); CmdArgs.push_back("-D_DLL"); CmdArgs.push_back("--dependent-lib=msvcrt"); - CmdArgs.push_back("--dependent-lib=Fortran_main.dynamic.lib"); + if (LinkFortranMain) + CmdArgs.push_back("--dependent-lib=Fortran_main.dynamic.lib"); CmdArgs.push_back("--dependent-lib=FortranRuntime.dynamic.lib"); CmdArgs.push_back("--dependent-lib=FortranDecimal.dynamic.lib"); break; @@ -273,7 +278,8 @@ static void processVSRuntimeLibrary(const ToolChain &TC, const ArgList &Args, CmdArgs.push_back("-D_DEBUG"); CmdArgs.push_back("-D_DLL"); CmdArgs.push_back("--dependent-lib=msvcrtd"); - CmdArgs.push_back("--dependent-lib=Fortran_main.dynamic_dbg.lib"); + if (LinkFortranMain) + CmdArgs.push_back("--dependent-lib=Fortran_main.dynamic_dbg.lib"); CmdArgs.push_back("--dependent-lib=FortranRuntime.dynamic_dbg.lib"); CmdArgs.push_back("--dependent-lib=FortranDecimal.dynamic_dbg.lib"); break; diff --git a/clang/lib/Driver/ToolChains/MSVC.cpp b/clang/lib/Driver/ToolChains/MSVC.cpp index 8a4a174c90ea..6d925555b7bb 100644 --- a/clang/lib/Driver/ToolChains/MSVC.cpp +++ b/clang/lib/Driver/ToolChains/MSVC.cpp @@ -787,11 +787,11 @@ VersionTuple MSVCToolChain::computeMSVCVersion(const Driver *D, if (MSVT.empty() && Args.hasFlag(options::OPT_fms_extensions, options::OPT_fno_ms_extensions, IsWindowsMSVC)) { - // -fms-compatibility-version=19.20 is default, aka 2019, 16.x + // -fms-compatibility-version=19.33 is default, aka 2022, 17.3 // NOTE: when changing this value, also update // clang/docs/CommandGuide/clang.rst and clang/docs/UsersManual.rst // accordingly. - MSVT = VersionTuple(19, 20); + MSVT = VersionTuple(19, 33); } return MSVT; } diff --git a/clang/lib/Driver/ToolChains/WebAssembly.cpp b/clang/lib/Driver/ToolChains/WebAssembly.cpp index f04018179a5d..f131b6cf3baf 100644 --- a/clang/lib/Driver/ToolChains/WebAssembly.cpp +++ b/clang/lib/Driver/ToolChains/WebAssembly.cpp @@ -143,7 +143,7 @@ void wasm::Linker::ConstructJob(Compilation &C, const JobAction &JA, // When optimizing, if wasm-opt is available, run it. std::string WasmOptPath; - if (Arg *A = Args.getLastArg(options::OPT_O_Group)) { + if (Args.getLastArg(options::OPT_O_Group)) { WasmOptPath = ToolChain.GetProgramPath("wasm-opt"); if (WasmOptPath == "wasm-opt") { WasmOptPath = {}; diff --git a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp index 931933b2bd1a..d9675b0c94de 100644 --- a/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp +++ b/clang/lib/ExtractAPI/Serialization/SymbolGraphSerializer.cpp @@ -199,9 +199,10 @@ StringRef getLanguageName(Language Lang) { return "objective-c"; case Language::CXX: return "c++"; + case Language::ObjCXX: + return "objective-c++"; // Unsupported language currently - case Language::ObjCXX: case Language::OpenCL: case Language::OpenCLCXX: case Language::CUDA: diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp index cb7dc5f3dd9b..9e4e939503df 100644 --- a/clang/lib/Format/ContinuationIndenter.cpp +++ b/clang/lib/Format/ContinuationIndenter.cpp @@ -1259,7 +1259,7 @@ unsigned ContinuationIndenter::getNewLineColumn(const LineState &State) { } if (Style.AlignAfterOpenBracket == FormatStyle::BAS_BlockIndent && (Current.is(tok::r_paren) || - (Current.is(tok::r_brace) && + (Current.is(tok::r_brace) && Current.MatchingParen && Current.MatchingParen->is(BK_BracedInit))) && State.Stack.size() > 1) { return State.Stack[State.Stack.size() - 2].LastSpace; diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp index b09487435adb..8feee7457fc3 100644 --- a/clang/lib/Format/Format.cpp +++ b/clang/lib/Format/Format.cpp @@ -3955,10 +3955,7 @@ llvm::Expected getStyle(StringRef StyleName, StringRef FileName, StringRef FallbackStyleName, StringRef Code, llvm::vfs::FileSystem *FS, bool AllowUnknownOptions) { - if (!FS) - FS = llvm::vfs::getRealFileSystem().get(); FormatStyle Style = getLLVMStyle(guessLanguage(FileName, Code)); - FormatStyle FallbackStyle = getNoStyle(); if (!getPredefinedStyle(FallbackStyleName, Style.Language, &FallbackStyle)) return make_string_error("Invalid fallback style: " + FallbackStyleName); @@ -3974,14 +3971,18 @@ llvm::Expected getStyle(StringRef StyleName, StringRef FileName, AllowUnknownOptions)) { return make_string_error("Error parsing -style: " + ec.message()); } - if (Style.InheritsParentConfig) { - ChildFormatTextToApply.emplace_back( - llvm::MemoryBuffer::getMemBuffer(StyleName, Source, false)); - } else { + + if (!Style.InheritsParentConfig) return Style; - } + + ChildFormatTextToApply.emplace_back( + llvm::MemoryBuffer::getMemBuffer(StyleName, Source, false)); } + if (!FS) + FS = llvm::vfs::getRealFileSystem().get(); + assert(FS); + // User provided clang-format file using -style=file:path/to/format/file. if (!Style.InheritsParentConfig && StyleName.starts_with_insensitive("file:")) { @@ -4015,18 +4016,12 @@ llvm::Expected getStyle(StringRef StyleName, StringRef FileName, return Style; } - // Reset possible inheritance - Style.InheritsParentConfig = false; - - // Look for .clang-format/_clang-format file in the file's parent directories. - SmallString<128> UnsuitableConfigFiles; SmallString<128> Path(FileName); if (std::error_code EC = FS->makeAbsolute(Path)) return make_string_error(EC.message()); - llvm::SmallVector FilesToLookFor; - FilesToLookFor.push_back(".clang-format"); - FilesToLookFor.push_back("_clang-format"); + // Reset possible inheritance + Style.InheritsParentConfig = false; auto dropDiagnosticHandler = [](const llvm::SMDiagnostic &, void *) {}; @@ -4040,9 +4035,14 @@ llvm::Expected getStyle(StringRef StyleName, StringRef FileName, } }; + // Look for .clang-format/_clang-format file in the file's parent directories. + llvm::SmallVector FilesToLookFor; + FilesToLookFor.push_back(".clang-format"); + FilesToLookFor.push_back("_clang-format"); + + SmallString<128> UnsuitableConfigFiles; for (StringRef Directory = Path; !Directory.empty(); Directory = llvm::sys::path::parent_path(Directory)) { - auto Status = FS->status(Directory); if (!Status || Status->getType() != llvm::sys::fs::file_type::directory_file) { @@ -4055,50 +4055,51 @@ llvm::Expected getStyle(StringRef StyleName, StringRef FileName, llvm::sys::path::append(ConfigFile, F); LLVM_DEBUG(llvm::dbgs() << "Trying " << ConfigFile << "...\n"); - Status = FS->status(ConfigFile.str()); - - if (Status && - (Status->getType() == llvm::sys::fs::file_type::regular_file)) { - llvm::ErrorOr> Text = - loadAndParseConfigFile(ConfigFile, FS, &Style, AllowUnknownOptions); - if (auto EC = Text.getError()) { - if (EC == ParseError::Unsuitable) { - if (!UnsuitableConfigFiles.empty()) - UnsuitableConfigFiles.append(", "); - UnsuitableConfigFiles.append(ConfigFile); - continue; - } + Status = FS->status(ConfigFile); + if (!Status || + Status->getType() != llvm::sys::fs::file_type::regular_file) { + continue; + } + + llvm::ErrorOr> Text = + loadAndParseConfigFile(ConfigFile, FS, &Style, AllowUnknownOptions); + if (auto EC = Text.getError()) { + if (EC != ParseError::Unsuitable) { return make_string_error("Error reading " + ConfigFile + ": " + EC.message()); } - LLVM_DEBUG(llvm::dbgs() - << "Using configuration file " << ConfigFile << "\n"); + if (!UnsuitableConfigFiles.empty()) + UnsuitableConfigFiles.append(", "); + UnsuitableConfigFiles.append(ConfigFile); + continue; + } - if (!Style.InheritsParentConfig) { - if (ChildFormatTextToApply.empty()) - return Style; + LLVM_DEBUG(llvm::dbgs() + << "Using configuration file " << ConfigFile << "\n"); + if (!Style.InheritsParentConfig) { + if (!ChildFormatTextToApply.empty()) { LLVM_DEBUG(llvm::dbgs() << "Applying child configurations\n"); applyChildFormatTexts(&Style); - - return Style; } + return Style; + } - LLVM_DEBUG(llvm::dbgs() << "Inherits parent configuration\n"); + LLVM_DEBUG(llvm::dbgs() << "Inherits parent configuration\n"); - // Reset inheritance of style - Style.InheritsParentConfig = false; + // Reset inheritance of style + Style.InheritsParentConfig = false; - ChildFormatTextToApply.emplace_back(std::move(*Text)); + ChildFormatTextToApply.emplace_back(std::move(*Text)); - // Breaking out of the inner loop, since we don't want to parse - // .clang-format AND _clang-format, if both exist. Then we continue the - // inner loop (parent directories) in search for the parent - // configuration. - break; - } + // Breaking out of the inner loop, since we don't want to parse + // .clang-format AND _clang-format, if both exist. Then we continue the + // outer loop (parent directories) in search for the parent + // configuration. + break; } } + if (!UnsuitableConfigFiles.empty()) { return make_string_error("Configuration file(s) do(es) not support " + getLanguageName(Style.Language) + ": " + diff --git a/clang/lib/Format/UnwrappedLineFormatter.cpp b/clang/lib/Format/UnwrappedLineFormatter.cpp index b4930c2e4621..56077499c39d 100644 --- a/clang/lib/Format/UnwrappedLineFormatter.cpp +++ b/clang/lib/Format/UnwrappedLineFormatter.cpp @@ -411,9 +411,16 @@ private: } } + const auto *LastNonComment = TheLine->getLastNonComment(); + assert(LastNonComment); + // FIXME: There are probably cases where we should use LastNonComment + // instead of TheLine->Last. + // Try to merge a function block with left brace unwrapped. - if (TheLine->Last->is(TT_FunctionLBrace) && TheLine->First != TheLine->Last) + if (LastNonComment->is(TT_FunctionLBrace) && + TheLine->First != LastNonComment) { return MergeShortFunctions ? tryMergeSimpleBlock(I, E, Limit) : 0; + } // Try to merge a control statement block with left brace unwrapped. if (TheLine->Last->is(tok::l_brace) && FirstNonComment != TheLine->Last && FirstNonComment->isOneOf(tok::kw_if, tok::kw_while, tok::kw_for, @@ -789,7 +796,8 @@ private: } } - if (Line.Last->is(tok::l_brace)) { + if (const auto *LastNonComment = Line.getLastNonComment(); + LastNonComment && LastNonComment->is(tok::l_brace)) { if (IsSplitBlock && Line.First == Line.Last && I > AnnotatedLines.begin() && (I[-1]->endsWith(tok::kw_else) || IsCtrlStmt(*I[-1]))) { @@ -805,7 +813,8 @@ private: if (ShouldMerge()) { // We merge empty blocks even if the line exceeds the column limit. - Tok->SpacesRequiredBefore = Style.SpaceInEmptyBlock ? 1 : 0; + Tok->SpacesRequiredBefore = + (Style.SpaceInEmptyBlock || Line.Last->is(tok::comment)) ? 1 : 0; Tok->CanBreakBefore = true; return 1; } else if (Limit != 0 && !Line.startsWithNamespace() && diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp index e5f8c0746a99..56bbef9697b6 100644 --- a/clang/lib/Frontend/CompilerInstance.cpp +++ b/clang/lib/Frontend/CompilerInstance.cpp @@ -2260,7 +2260,7 @@ GlobalModuleIndex *CompilerInstance::loadGlobalModuleIndex( for (ModuleMap::module_iterator I = MMap.module_begin(), E = MMap.module_end(); I != E; ++I) { Module *TheModule = I->second; - const FileEntry *Entry = TheModule->getASTFile(); + OptionalFileEntryRef Entry = TheModule->getASTFile(); if (!Entry) { SmallVector, 2> Path; Path.push_back(std::make_pair( diff --git a/clang/lib/Frontend/CompilerInvocation.cpp b/clang/lib/Frontend/CompilerInvocation.cpp index 56de0f75928c..b33bdad2ad81 100644 --- a/clang/lib/Frontend/CompilerInvocation.cpp +++ b/clang/lib/Frontend/CompilerInvocation.cpp @@ -4365,6 +4365,9 @@ static void GeneratePreprocessorArgs(const PreprocessorOptions &Opts, if (Opts.SourceDateEpoch) GenerateArg(Consumer, OPT_source_date_epoch, Twine(*Opts.SourceDateEpoch)); + if (Opts.DefineTargetOSMacros) + GenerateArg(Consumer, OPT_fdefine_target_os_macros); + // Don't handle LexEditorPlaceholders. It is implied by the action that is // generated elsewhere. } @@ -4463,6 +4466,10 @@ static bool ParsePreprocessorArgs(PreprocessorOptions &Opts, ArgList &Args, if (isStrictlyPreprocessorAction(Action)) Opts.LexEditorPlaceholders = false; + Opts.DefineTargetOSMacros = + Args.hasFlag(OPT_fdefine_target_os_macros, + OPT_fno_define_target_os_macros, Opts.DefineTargetOSMacros); + return Diags.getNumErrors() == NumErrorsBefore; } diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp index 2afcf1cf9f68..c1d6e7145536 100644 --- a/clang/lib/Frontend/FrontendActions.cpp +++ b/clang/lib/Frontend/FrontendActions.cpp @@ -258,6 +258,16 @@ bool GenerateModuleInterfaceAction::BeginSourceFileAction( return GenerateModuleAction::BeginSourceFileAction(CI); } +std::unique_ptr +GenerateModuleInterfaceAction::CreateASTConsumer(CompilerInstance &CI, + StringRef InFile) { + CI.getHeaderSearchOpts().ModulesSkipDiagnosticOptions = true; + CI.getHeaderSearchOpts().ModulesSkipHeaderSearchPaths = true; + CI.getHeaderSearchOpts().ModulesSkipPragmaDiagnosticMappings = true; + + return GenerateModuleAction::CreateASTConsumer(CI, InFile); +} + std::unique_ptr GenerateModuleInterfaceAction::CreateOutputFile(CompilerInstance &CI, StringRef InFile) { diff --git a/clang/lib/Frontend/InitPreprocessor.cpp b/clang/lib/Frontend/InitPreprocessor.cpp index 17948dcebd7e..d83128adb511 100644 --- a/clang/lib/Frontend/InitPreprocessor.cpp +++ b/clang/lib/Frontend/InitPreprocessor.cpp @@ -809,6 +809,13 @@ static void InitializePredefinedMacros(const TargetInfo &TI, Builder.defineMacro("__ATOMIC_ACQ_REL", "4"); Builder.defineMacro("__ATOMIC_SEQ_CST", "5"); + // Define macros for the clang atomic scopes. + Builder.defineMacro("__MEMORY_SCOPE_SYSTEM", "0"); + Builder.defineMacro("__MEMORY_SCOPE_DEVICE", "1"); + Builder.defineMacro("__MEMORY_SCOPE_WRKGRP", "2"); + Builder.defineMacro("__MEMORY_SCOPE_WVFRNT", "3"); + Builder.defineMacro("__MEMORY_SCOPE_SINGLE", "4"); + // Define macros for the OpenCL memory scope. // The values should match AtomicScopeOpenCLModel::ID enum. static_assert( @@ -1344,6 +1351,15 @@ static void InitializePredefinedMacros(const TargetInfo &TI, if (TI.getTriple().isOSBinFormatELF()) Builder.defineMacro("__ELF__"); + // Target OS macro definitions. + if (PPOpts.DefineTargetOSMacros) { + const llvm::Triple &Triple = TI.getTriple(); +#define TARGET_OS(Name, Predicate) \ + Builder.defineMacro(#Name, (Predicate) ? "1" : "0"); +#include "clang/Basic/TargetOSMacros.def" +#undef TARGET_OS + } + // Get other target #defines. TI.getTargetDefines(LangOpts, Builder); } diff --git a/clang/lib/Lex/HeaderSearch.cpp b/clang/lib/Lex/HeaderSearch.cpp index cf1c0cc5284f..c03cf19688d9 100644 --- a/clang/lib/Lex/HeaderSearch.cpp +++ b/clang/lib/Lex/HeaderSearch.cpp @@ -105,7 +105,7 @@ void HeaderSearch::PrintStats() { void HeaderSearch::SetSearchPaths( std::vector dirs, unsigned int angledDirIdx, - unsigned int systemDirIdx, bool noCurDirSearch, + unsigned int systemDirIdx, llvm::DenseMap searchDirToHSEntry) { assert(angledDirIdx <= systemDirIdx && systemDirIdx <= dirs.size() && "Directory indices are unordered"); @@ -113,7 +113,6 @@ void HeaderSearch::SetSearchPaths( SearchDirsUsage.assign(SearchDirs.size(), false); AngledDirIdx = angledDirIdx; SystemDirIdx = systemDirIdx; - NoCurDirSearch = noCurDirSearch; SearchDirToHSEntry = std::move(searchDirToHSEntry); //LookupFileCache.clear(); indexInitialHeaderMaps(); @@ -904,12 +903,12 @@ OptionalFileEntryRef HeaderSearch::LookupFile( ModuleMap::KnownHeader MSSuggestedModule; OptionalFileEntryRef MSFE; - // Unless disabled, check to see if the file is in the #includer's - // directory. This cannot be based on CurDir, because each includer could be - // a #include of a subdirectory (#include "foo/bar.h") and a subsequent - // include of "baz.h" should resolve to "whatever/foo/baz.h". - // This search is not done for <> headers. - if (!Includers.empty() && !isAngled && !NoCurDirSearch) { + // Check to see if the file is in the #includer's directory. This cannot be + // based on CurDir, because each includer could be a #include of a + // subdirectory (#include "foo/bar.h") and a subsequent include of "baz.h" + // should resolve to "whatever/foo/baz.h". This search is not done for <> + // headers. + if (!Includers.empty() && !isAngled) { SmallString<1024> TmpDir; bool First = true; for (const auto &IncluderAndDir : Includers) { diff --git a/clang/lib/Lex/InitHeaderSearch.cpp b/clang/lib/Lex/InitHeaderSearch.cpp index 1f4c7a6d67b7..5b1b7c859c85 100644 --- a/clang/lib/Lex/InitHeaderSearch.cpp +++ b/clang/lib/Lex/InitHeaderSearch.cpp @@ -513,9 +513,8 @@ void InitHeaderSearch::Realize(const LangOptions &Lang) { unsigned NonSystemRemoved = RemoveDuplicates(SearchList, NumQuoted, Verbose); NumAngled -= NonSystemRemoved; - bool DontSearchCurDir = false; // TODO: set to true if -I- is set? Headers.SetSearchPaths(extractLookups(SearchList), NumQuoted, NumAngled, - DontSearchCurDir, mapToUserEntries(SearchList)); + mapToUserEntries(SearchList)); Headers.SetSystemHeaderPrefixes(SystemHeaderPrefixes); diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp index 1d67e275cb47..d35c282543c5 100644 --- a/clang/lib/Lex/ModuleMap.cpp +++ b/clang/lib/Lex/ModuleMap.cpp @@ -1067,9 +1067,7 @@ Module *ModuleMap::inferFrameworkModule(DirectoryEntryRef FrameworkDir, if (!canInfer) return nullptr; } else { - OptionalFileEntryRefDegradesToFileEntryPtr ModuleMapRef = - getModuleMapFileForUniquing(Parent); - ModuleMapFile = ModuleMapRef; + ModuleMapFile = getModuleMapFileForUniquing(Parent); } // Look for an umbrella header. @@ -1866,7 +1864,7 @@ void ModuleMapParser::diagnosePrivateModules(SourceLocation ExplicitLoc, continue; SmallString<128> FullName(ActiveModule->getFullModuleName()); - if (!FullName.startswith(M->Name) && !FullName.endswith("Private")) + if (!FullName.starts_with(M->Name) && !FullName.ends_with("Private")) continue; SmallString<128> FixedPrivModDecl; SmallString<128> Canonical(M->Name); diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp index 956e2276f25b..14003480d7fa 100644 --- a/clang/lib/Lex/PPDirectives.cpp +++ b/clang/lib/Lex/PPDirectives.cpp @@ -1934,7 +1934,8 @@ Preprocessor::getIncludeNextStart(const Token &IncludeNextTok) const { // Start looking up in the directory *after* the one in which the current // file would be found, if any. assert(CurPPLexer && "#include_next directive in macro?"); - LookupFromFile = CurPPLexer->getFileEntry(); + if (auto FE = CurPPLexer->getFileEntry()) + LookupFromFile = *FE; Lookup = nullptr; } else if (!Lookup) { // The current file was not found by walking the include path. Either it diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp index 35ab42cb6b5e..499813f8ab7d 100644 --- a/clang/lib/Lex/Pragma.cpp +++ b/clang/lib/Lex/Pragma.cpp @@ -548,7 +548,7 @@ void Preprocessor::HandlePragmaDependency(Token &DependencyTok) { return; } - const FileEntry *CurFile = getCurrentFileLexer()->getFileEntry(); + OptionalFileEntryRef CurFile = getCurrentFileLexer()->getFileEntry(); // If this file is older than the file it depends on, emit a diagnostic. if (CurFile && CurFile->getModificationTime() < File->getModificationTime()) { diff --git a/clang/lib/Lex/PreprocessorLexer.cpp b/clang/lib/Lex/PreprocessorLexer.cpp index 23c80d375214..7551ba235fe9 100644 --- a/clang/lib/Lex/PreprocessorLexer.cpp +++ b/clang/lib/Lex/PreprocessorLexer.cpp @@ -47,7 +47,6 @@ void PreprocessorLexer::LexIncludeFilename(Token &FilenameTok) { /// getFileEntry - Return the FileEntry corresponding to this FileID. Like /// getFileID(), this only works for lexers with attached preprocessors. -OptionalFileEntryRefDegradesToFileEntryPtr -PreprocessorLexer::getFileEntry() const { +OptionalFileEntryRef PreprocessorLexer::getFileEntry() const { return PP->getSourceManager().getFileEntryRefForID(getFileID()); } diff --git a/clang/lib/Parse/ParseOpenACC.cpp b/clang/lib/Parse/ParseOpenACC.cpp index 71cb665a5632..f7f096762e91 100644 --- a/clang/lib/Parse/ParseOpenACC.cpp +++ b/clang/lib/Parse/ParseOpenACC.cpp @@ -56,6 +56,7 @@ OpenACCDirectiveKindEx getOpenACCDirectiveKind(Token Tok) { .Case("shutdown", OpenACCDirectiveKind::Shutdown) .Case("set", OpenACCDirectiveKind::Shutdown) .Case("update", OpenACCDirectiveKind::Update) + .Case("wait", OpenACCDirectiveKind::Wait) .Default(OpenACCDirectiveKind::Invalid); if (DirKind != OpenACCDirectiveKind::Invalid) @@ -82,6 +83,27 @@ OpenACCAtomicKind getOpenACCAtomicKind(Token Tok) { .Default(OpenACCAtomicKind::Invalid); } +enum class OpenACCSpecialTokenKind { + ReadOnly, + DevNum, + Queues, +}; + +bool isOpenACCSpecialToken(OpenACCSpecialTokenKind Kind, Token Tok) { + if (!Tok.is(tok::identifier)) + return false; + + switch (Kind) { + case OpenACCSpecialTokenKind::ReadOnly: + return Tok.getIdentifierInfo()->isStr("readonly"); + case OpenACCSpecialTokenKind::DevNum: + return Tok.getIdentifierInfo()->isStr("devnum"); + case OpenACCSpecialTokenKind::Queues: + return Tok.getIdentifierInfo()->isStr("queues"); + } + llvm_unreachable("Unknown 'Kind' Passed"); +} + bool isOpenACCDirectiveKind(OpenACCDirectiveKind Kind, Token Tok) { if (!Tok.is(tok::identifier)) return false; @@ -123,6 +145,8 @@ bool isOpenACCDirectiveKind(OpenACCDirectiveKind Kind, Token Tok) { return Tok.getIdentifierInfo()->isStr("set"); case OpenACCDirectiveKind::Update: return Tok.getIdentifierInfo()->isStr("update"); + case OpenACCDirectiveKind::Wait: + return Tok.getIdentifierInfo()->isStr("wait"); case OpenACCDirectiveKind::Invalid: return false; } @@ -182,7 +206,7 @@ OpenACCDirectiveKind ParseOpenACCDirectiveKind(Parser &P) { // Just #pragma acc can get us immediately to the end, make sure we don't // introspect on the spelling before then. - if (FirstTok.isAnnotation()) { + if (FirstTok.isNot(tok::identifier)) { P.Diag(FirstTok, diag::err_acc_missing_directive); return OpenACCDirectiveKind::Invalid; } @@ -200,11 +224,8 @@ OpenACCDirectiveKind ParseOpenACCDirectiveKind(Parser &P) { if (ExDirKind >= OpenACCDirectiveKindEx::Invalid) { switch (ExDirKind) { case OpenACCDirectiveKindEx::Invalid: { - if (!FirstTok.is(tok::identifier)) - P.Diag(FirstTok, diag::err_expected) << tok::identifier; - else - P.Diag(FirstTok, diag::err_acc_invalid_directive) - << 0 << FirstTok.getIdentifierInfo(); + P.Diag(FirstTok, diag::err_acc_invalid_directive) + << 0 << FirstTok.getIdentifierInfo(); return OpenACCDirectiveKind::Invalid; } case OpenACCDirectiveKindEx::Enter: @@ -251,6 +272,67 @@ void ParseOpenACCClauseList(Parser &P) { } // namespace +/// OpenACC 3.3, section 2.16: +/// In this section and throughout the specification, the term wait-argument +/// means: +/// [ devnum : int-expr : ] [ queues : ] async-argument-list +bool Parser::ParseOpenACCWaitArgument() { + // [devnum : int-expr : ] + if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::DevNum, Tok) && + NextToken().is(tok::colon)) { + // Consume devnum. + ConsumeToken(); + // Consume colon. + ConsumeToken(); + + ExprResult IntExpr = + getActions().CorrectDelayedTyposInExpr(ParseAssignmentExpression()); + if (IntExpr.isInvalid()) + return true; + + if (ExpectAndConsume(tok::colon)) + return true; + } + + // [ queues : ] + if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::Queues, Tok) && + NextToken().is(tok::colon)) { + // Consume queues. + ConsumeToken(); + // Consume colon. + ConsumeToken(); + } + + // OpenACC 3.3, section 2.16: + // the term 'async-argument' means a nonnegative scalar integer expression, or + // one of the special values 'acc_async_noval' or 'acc_async_sync', as defined + // in the C header file and the Fortran opacc module. + // + // We are parsing this simply as list of assignment expressions (to avoid + // comma being troublesome), and will ensure it is an integral type. The + // 'special' types are defined as macros, so we can't really check those + // (other than perhaps as values at one point?), but the standard does say it + // is implementation-defined to use any other negative value. + // + // + bool FirstArg = true; + while (!getCurToken().isOneOf(tok::r_paren, tok::annot_pragma_openacc_end)) { + if (!FirstArg) { + if (ExpectAndConsume(tok::comma)) + return true; + } + FirstArg = false; + + ExprResult CurArg = + getActions().CorrectDelayedTyposInExpr(ParseAssignmentExpression()); + + if (CurArg.isInvalid()) + return true; + } + + return false; +} + ExprResult Parser::ParseOpenACCIDExpression() { ExprResult Res; if (getLangOpts().CPlusPlus) { @@ -340,8 +422,7 @@ void Parser::ParseOpenACCCacheVarList() { // specifications. First, see if we have `readonly:`, else we back-out and // treat it like the beginning of a reference to a potentially-existing // `readonly` variable. - if (getCurToken().is(tok::identifier) && - getCurToken().getIdentifierInfo()->isStr("readonly") && + if (isOpenACCSpecialToken(OpenACCSpecialTokenKind::ReadOnly, Tok) && NextToken().is(tok::colon)) { // Consume both tokens. ConsumeToken(); @@ -399,6 +480,13 @@ void Parser::ParseOpenACCDirective() { // so we can always consume the close. T.consumeClose(); break; + case OpenACCDirectiveKind::Wait: + // OpenACC has an optional paren-wrapped 'wait-argument'. + if (ParseOpenACCWaitArgument()) + T.skipToEnd(); + else + T.consumeClose(); + break; } } else if (DirKind == OpenACCDirectiveKind::Cache) { // Cache's paren var-list is required, so error here if it isn't provided. diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp index efdf7c90f977..730ac1a0fee5 100644 --- a/clang/lib/Parse/ParsePragma.cpp +++ b/clang/lib/Parse/ParsePragma.cpp @@ -137,7 +137,20 @@ struct PragmaSTDC_CX_LIMITED_RANGEHandler : public PragmaHandler { void HandlePragma(Preprocessor &PP, PragmaIntroducer Introducer, Token &Tok) override { tok::OnOffSwitch OOS; - PP.LexOnOffSwitch(OOS); + if (PP.LexOnOffSwitch(OOS)) + return; + + MutableArrayRef Toks( + PP.getPreprocessorAllocator().Allocate(1), 1); + + Toks[0].startToken(); + Toks[0].setKind(tok::annot_pragma_cx_limited_range); + Toks[0].setLocation(Tok.getLocation()); + Toks[0].setAnnotationEndLoc(Tok.getLocation()); + Toks[0].setAnnotationValue( + reinterpret_cast(static_cast(OOS))); + PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true, + /*IsReinject=*/false); } }; @@ -888,6 +901,31 @@ void Parser::HandlePragmaFEnvRound() { Actions.ActOnPragmaFEnvRound(PragmaLoc, RM); } +void Parser::HandlePragmaCXLimitedRange() { + assert(Tok.is(tok::annot_pragma_cx_limited_range)); + tok::OnOffSwitch OOS = static_cast( + reinterpret_cast(Tok.getAnnotationValue())); + + LangOptions::ComplexRangeKind Range; + switch (OOS) { + case tok::OOS_ON: + Range = LangOptions::CX_Limited; + break; + case tok::OOS_OFF: + Range = LangOptions::CX_Full; + break; + case tok::OOS_DEFAULT: + // According to ISO C99 standard chapter 7.3.4, the default value + // for the pragma is ``off'. -fcx-limited-range and -fcx-fortran-rules + // control the default value of these pragmas. + Range = getLangOpts().getComplexRange(); + break; + } + + SourceLocation PragmaLoc = ConsumeAnnotationToken(); + Actions.ActOnPragmaCXLimitedRange(PragmaLoc, Range); +} + StmtResult Parser::HandlePragmaCaptured() { assert(Tok.is(tok::annot_pragma_captured)); diff --git a/clang/lib/Parse/ParseStmt.cpp b/clang/lib/Parse/ParseStmt.cpp index 918afdc2baea..d0ff33bd1379 100644 --- a/clang/lib/Parse/ParseStmt.cpp +++ b/clang/lib/Parse/ParseStmt.cpp @@ -444,6 +444,14 @@ Retry: ConsumeAnnotationToken(); return StmtError(); + case tok::annot_pragma_cx_limited_range: + ProhibitAttributes(CXX11Attrs); + ProhibitAttributes(GNUAttrs); + Diag(Tok, diag::err_pragma_file_or_compound_scope) + << "STDC CX_LIMITED_RANGE"; + ConsumeAnnotationToken(); + return StmtError(); + case tok::annot_pragma_float_control: ProhibitAttributes(CXX11Attrs); ProhibitAttributes(GNUAttrs); @@ -1066,6 +1074,9 @@ void Parser::ParseCompoundStatementLeadingPragmas() { case tok::annot_pragma_fenv_round: HandlePragmaFEnvRound(); break; + case tok::annot_pragma_cx_limited_range: + HandlePragmaCXLimitedRange(); + break; case tok::annot_pragma_float_control: HandlePragmaFloatControl(); break; diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp index 1baeb2aeb021..ec67faf7dcaf 100644 --- a/clang/lib/Parse/Parser.cpp +++ b/clang/lib/Parse/Parser.cpp @@ -844,6 +844,9 @@ Parser::ParseExternalDeclaration(ParsedAttributes &Attrs, case tok::annot_pragma_fenv_round: HandlePragmaFEnvRound(); return nullptr; + case tok::annot_pragma_cx_limited_range: + HandlePragmaCXLimitedRange(); + return nullptr; case tok::annot_pragma_float_control: HandlePragmaFloatControl(); return nullptr; diff --git a/clang/lib/Sema/HLSLExternalSemaSource.cpp b/clang/lib/Sema/HLSLExternalSemaSource.cpp index 8de144486c91..1a1febf7a352 100644 --- a/clang/lib/Sema/HLSLExternalSemaSource.cpp +++ b/clang/lib/Sema/HLSLExternalSemaSource.cpp @@ -116,11 +116,11 @@ struct BuiltinTypeDeclBuilder { } BuiltinTypeDeclBuilder &annotateResourceClass(ResourceClass RC, - ResourceKind RK) { + ResourceKind RK, bool IsROV) { if (Record->isCompleteDefinition()) return *this; - Record->addAttr( - HLSLResourceAttr::CreateImplicit(Record->getASTContext(), RC, RK)); + Record->addAttr(HLSLResourceAttr::CreateImplicit(Record->getASTContext(), + RC, RK, IsROV)); return *this; } @@ -306,6 +306,7 @@ struct BuiltinTypeDeclBuilder { } TemplateParameterListBuilder addTemplateArgumentList(); + BuiltinTypeDeclBuilder &addSimpleTemplateParams(ArrayRef Names); }; struct TemplateParameterListBuilder { @@ -360,11 +361,19 @@ struct TemplateParameterListBuilder { return Builder; } }; +} // namespace TemplateParameterListBuilder BuiltinTypeDeclBuilder::addTemplateArgumentList() { return TemplateParameterListBuilder(*this); } -} // namespace + +BuiltinTypeDeclBuilder & +BuiltinTypeDeclBuilder::addSimpleTemplateParams(ArrayRef Names) { + TemplateParameterListBuilder Builder = this->addTemplateArgumentList(); + for (StringRef Name : Names) + Builder.addTypeParameter(Name); + return Builder.finalizeTemplateArgs(); +} HLSLExternalSemaSource::~HLSLExternalSemaSource() {} @@ -390,7 +399,7 @@ void HLSLExternalSemaSource::InitializeSema(Sema &S) { // Force external decls in the HLSL namespace to load from the PCH. (void)HLSLNamespace->getCanonicalDecl()->decls_begin(); defineTrivialHLSLTypes(); - forwardDeclareHLSLTypes(); + defineHLSLTypesWithForwardDeclarations(); // This adds a `using namespace hlsl` directive. In DXC, we don't put HLSL's // built in types inside a namespace, but we are planning to change that in @@ -467,18 +476,43 @@ void HLSLExternalSemaSource::defineTrivialHLSLTypes() { .Record; } -void HLSLExternalSemaSource::forwardDeclareHLSLTypes() { +/// Set up common members and attributes for buffer types +static BuiltinTypeDeclBuilder setupBufferType(CXXRecordDecl *Decl, Sema &S, + ResourceClass RC, ResourceKind RK, + bool IsROV) { + return BuiltinTypeDeclBuilder(Decl) + .addHandleMember() + .addDefaultHandleConstructor(S, RC) + .annotateResourceClass(RC, RK, IsROV); +} + +void HLSLExternalSemaSource::defineHLSLTypesWithForwardDeclarations() { CXXRecordDecl *Decl; Decl = BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "RWBuffer") - .addTemplateArgumentList() - .addTypeParameter("element_type", SemaPtr->getASTContext().FloatTy) - .finalizeTemplateArgs() + .addSimpleTemplateParams({"element_type"}) .Record; - if (!Decl->isCompleteDefinition()) - Completions.insert( - std::make_pair(Decl->getCanonicalDecl(), - std::bind(&HLSLExternalSemaSource::completeBufferType, - this, std::placeholders::_1))); + onCompletion(Decl, [this](CXXRecordDecl *Decl) { + setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, + ResourceKind::TypedBuffer, /*IsROV=*/false) + .addArraySubscriptOperators() + .completeDefinition(); + }); + + Decl = + BuiltinTypeDeclBuilder(*SemaPtr, HLSLNamespace, "RasterizerOrderedBuffer") + .addSimpleTemplateParams({"element_type"}) + .Record; + onCompletion(Decl, [this](CXXRecordDecl *Decl) { + setupBufferType(Decl, *SemaPtr, ResourceClass::UAV, + ResourceKind::TypedBuffer, /*IsROV=*/true) + .addArraySubscriptOperators() + .completeDefinition(); + }); +} + +void HLSLExternalSemaSource::onCompletion(CXXRecordDecl *Record, + CompletionFunction Fn) { + Completions.insert(std::make_pair(Record->getCanonicalDecl(), Fn)); } void HLSLExternalSemaSource::CompleteType(TagDecl *Tag) { @@ -496,12 +530,3 @@ void HLSLExternalSemaSource::CompleteType(TagDecl *Tag) { return; It->second(Record); } - -void HLSLExternalSemaSource::completeBufferType(CXXRecordDecl *Record) { - BuiltinTypeDeclBuilder(Record) - .addHandleMember() - .addDefaultHandleConstructor(*SemaPtr, ResourceClass::UAV) - .addArraySubscriptOperators() - .annotateResourceClass(ResourceClass::UAV, ResourceKind::TypedBuffer) - .completeDefinition(); -} diff --git a/clang/lib/Sema/Sema.cpp b/clang/lib/Sema/Sema.cpp index 2c7ecf4610de..22929aa6316d 100644 --- a/clang/lib/Sema/Sema.cpp +++ b/clang/lib/Sema/Sema.cpp @@ -2077,7 +2077,7 @@ void Sema::checkTypeSupport(QualType Ty, SourceLocation Loc, ValueDecl *D) { targetDiag(D->getLocation(), diag::note_defined_here, FD) << D; } - if (TI.hasRISCVVTypes() && Ty->isRVVType()) + if (TI.hasRISCVVTypes() && Ty->isRVVSizelessBuiltinType()) checkRVVTypeSupport(Ty, Loc, D); // Don't allow SVE types in functions without a SVE target. diff --git a/clang/lib/Sema/SemaAttr.cpp b/clang/lib/Sema/SemaAttr.cpp index 79271c872627..0dcf42e48997 100644 --- a/clang/lib/Sema/SemaAttr.cpp +++ b/clang/lib/Sema/SemaAttr.cpp @@ -1352,6 +1352,14 @@ void Sema::ActOnPragmaFEnvAccess(SourceLocation Loc, bool IsEnabled) { CurFPFeatures = NewFPFeatures.applyOverrides(getLangOpts()); } +void Sema::ActOnPragmaCXLimitedRange(SourceLocation Loc, + LangOptions::ComplexRangeKind Range) { + FPOptionsOverride NewFPFeatures = CurFPFeatureOverrides(); + NewFPFeatures.setComplexRangeOverride(Range); + FpPragmaStack.Act(Loc, PSK_Set, StringRef(), NewFPFeatures); + CurFPFeatures = NewFPFeatures.applyOverrides(getLangOpts()); +} + void Sema::ActOnPragmaFPExceptions(SourceLocation Loc, LangOptions::FPExceptionModeKind FPE) { setExceptionMode(Loc, FPE); diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp index fc6ee6b2c5ab..cdb6e9584e95 100644 --- a/clang/lib/Sema/SemaChecking.cpp +++ b/clang/lib/Sema/SemaChecking.cpp @@ -5082,12 +5082,10 @@ static bool CheckInvalidVLENandLMUL(const TargetInfo &TI, CallExpr *TheCall, assert((EGW == 128 || EGW == 256) && "EGW can only be 128 or 256 bits"); // LMUL * VLEN >= EGW - unsigned ElemSize = Type->isRVVType(32, false) ? 32 : 64; - unsigned MinElemCount = Type->isRVVType(1) ? 1 - : Type->isRVVType(2) ? 2 - : Type->isRVVType(4) ? 4 - : Type->isRVVType(8) ? 8 - : 16; + ASTContext::BuiltinVectorTypeInfo Info = + S.Context.getBuiltinVectorTypeInfo(Type->castAs()); + unsigned ElemSize = S.Context.getTypeSize(Info.ElementType); + unsigned MinElemCount = Info.EC.getKnownMinValue(); unsigned EGS = EGW / ElemSize; // If EGS is less than or equal to the minimum number of elements, then the @@ -5215,15 +5213,13 @@ bool Sema::CheckRISCVBuiltinFunctionCall(const TargetInfo &TI, case RISCVVector::BI__builtin_rvv_vsmul_vx_tum: case RISCVVector::BI__builtin_rvv_vsmul_vv_tumu: case RISCVVector::BI__builtin_rvv_vsmul_vx_tumu: { - bool RequireV = false; - for (unsigned ArgNum = 0; ArgNum < TheCall->getNumArgs(); ++ArgNum) - RequireV |= TheCall->getArg(ArgNum)->getType()->isRVVType( - /* Bitwidth */ 64, /* IsFloat */ false); + ASTContext::BuiltinVectorTypeInfo Info = Context.getBuiltinVectorTypeInfo( + TheCall->getType()->castAs()); - if (RequireV && !TI.hasFeature("v")) + if (Context.getTypeSize(Info.ElementType) == 64 && !TI.hasFeature("v")) return Diag(TheCall->getBeginLoc(), diag::err_riscv_builtin_requires_extension) - << /* IsExtension */ false << TheCall->getSourceRange() << "v"; + << /* IsExtension */ true << TheCall->getSourceRange() << "v"; break; } @@ -5983,7 +5979,7 @@ bool Sema::CheckRISCVBuiltinFunctionCall(const TargetInfo &TI, ValType = ValType.getUnqualifiedType(); if (!ValType->isIntegerType() && !ValType->isAnyPointerType() && !ValType->isBlockPointerType() && !ValType->isFloatingType() && - !ValType->isVectorType() && !ValType->isRVVType()) { + !ValType->isVectorType() && !ValType->isRVVSizelessBuiltinType()) { Diag(DRE->getBeginLoc(), diag::err_nontemporal_builtin_must_be_pointer_intfltptr_or_vector) << PointerArg->getType() << PointerArg->getSourceRange(); @@ -7653,6 +7649,8 @@ static bool isValidOrderingForOp(int64_t Ordering, AtomicExpr::AtomicOp Op) { case AtomicExpr::AO__hip_atomic_load: case AtomicExpr::AO__atomic_load_n: case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__scoped_atomic_load_n: + case AtomicExpr::AO__scoped_atomic_load: return OrderingCABI != llvm::AtomicOrderingCABI::release && OrderingCABI != llvm::AtomicOrderingCABI::acq_rel; @@ -7661,6 +7659,8 @@ static bool isValidOrderingForOp(int64_t Ordering, AtomicExpr::AtomicOp Op) { case AtomicExpr::AO__hip_atomic_store: case AtomicExpr::AO__atomic_store: case AtomicExpr::AO__atomic_store_n: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_store_n: return OrderingCABI != llvm::AtomicOrderingCABI::consume && OrderingCABI != llvm::AtomicOrderingCABI::acquire && OrderingCABI != llvm::AtomicOrderingCABI::acq_rel; @@ -7737,13 +7737,19 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, Op <= AtomicExpr::AO__opencl_atomic_fetch_max; bool IsHIP = Op >= AtomicExpr::AO__hip_atomic_load && Op <= AtomicExpr::AO__hip_atomic_fetch_max; + bool IsScoped = Op >= AtomicExpr::AO__scoped_atomic_load && + Op <= AtomicExpr::AO__scoped_atomic_fetch_max; bool IsC11 = (Op >= AtomicExpr::AO__c11_atomic_init && Op <= AtomicExpr::AO__c11_atomic_fetch_min) || IsOpenCL; bool IsN = Op == AtomicExpr::AO__atomic_load_n || Op == AtomicExpr::AO__atomic_store_n || Op == AtomicExpr::AO__atomic_exchange_n || - Op == AtomicExpr::AO__atomic_compare_exchange_n; + Op == AtomicExpr::AO__atomic_compare_exchange_n || + Op == AtomicExpr::AO__scoped_atomic_load_n || + Op == AtomicExpr::AO__scoped_atomic_store_n || + Op == AtomicExpr::AO__scoped_atomic_exchange_n || + Op == AtomicExpr::AO__scoped_atomic_compare_exchange_n; // Bit mask for extra allowed value types other than integers for atomic // arithmetic operations. Add/sub allow pointer and floating point. Min/max // allow floating point. @@ -7764,10 +7770,12 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__opencl_atomic_load: case AtomicExpr::AO__hip_atomic_load: case AtomicExpr::AO__atomic_load_n: + case AtomicExpr::AO__scoped_atomic_load_n: Form = Load; break; case AtomicExpr::AO__atomic_load: + case AtomicExpr::AO__scoped_atomic_load: Form = LoadCopy; break; @@ -7776,12 +7784,18 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__hip_atomic_store: case AtomicExpr::AO__atomic_store: case AtomicExpr::AO__atomic_store_n: + case AtomicExpr::AO__scoped_atomic_store: + case AtomicExpr::AO__scoped_atomic_store_n: Form = Copy; break; case AtomicExpr::AO__atomic_fetch_add: case AtomicExpr::AO__atomic_fetch_sub: case AtomicExpr::AO__atomic_add_fetch: case AtomicExpr::AO__atomic_sub_fetch: + case AtomicExpr::AO__scoped_atomic_fetch_add: + case AtomicExpr::AO__scoped_atomic_fetch_sub: + case AtomicExpr::AO__scoped_atomic_add_fetch: + case AtomicExpr::AO__scoped_atomic_sub_fetch: case AtomicExpr::AO__c11_atomic_fetch_add: case AtomicExpr::AO__c11_atomic_fetch_sub: case AtomicExpr::AO__opencl_atomic_fetch_add: @@ -7795,6 +7809,10 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__atomic_fetch_min: case AtomicExpr::AO__atomic_max_fetch: case AtomicExpr::AO__atomic_min_fetch: + case AtomicExpr::AO__scoped_atomic_fetch_max: + case AtomicExpr::AO__scoped_atomic_fetch_min: + case AtomicExpr::AO__scoped_atomic_max_fetch: + case AtomicExpr::AO__scoped_atomic_min_fetch: case AtomicExpr::AO__c11_atomic_fetch_max: case AtomicExpr::AO__c11_atomic_fetch_min: case AtomicExpr::AO__opencl_atomic_fetch_max: @@ -7822,6 +7840,14 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__atomic_or_fetch: case AtomicExpr::AO__atomic_xor_fetch: case AtomicExpr::AO__atomic_nand_fetch: + case AtomicExpr::AO__scoped_atomic_fetch_and: + case AtomicExpr::AO__scoped_atomic_fetch_or: + case AtomicExpr::AO__scoped_atomic_fetch_xor: + case AtomicExpr::AO__scoped_atomic_fetch_nand: + case AtomicExpr::AO__scoped_atomic_and_fetch: + case AtomicExpr::AO__scoped_atomic_or_fetch: + case AtomicExpr::AO__scoped_atomic_xor_fetch: + case AtomicExpr::AO__scoped_atomic_nand_fetch: Form = Arithmetic; break; @@ -7829,10 +7855,12 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__hip_atomic_exchange: case AtomicExpr::AO__opencl_atomic_exchange: case AtomicExpr::AO__atomic_exchange_n: + case AtomicExpr::AO__scoped_atomic_exchange_n: Form = Xchg; break; case AtomicExpr::AO__atomic_exchange: + case AtomicExpr::AO__scoped_atomic_exchange: Form = GNUXchg; break; @@ -7847,12 +7875,15 @@ ExprResult Sema::BuildAtomicExpr(SourceRange CallRange, SourceRange ExprRange, case AtomicExpr::AO__atomic_compare_exchange: case AtomicExpr::AO__atomic_compare_exchange_n: + case AtomicExpr::AO__scoped_atomic_compare_exchange: + case AtomicExpr::AO__scoped_atomic_compare_exchange_n: Form = GNUCmpXchg; break; } unsigned AdjustedNumArgs = NumArgs[Form]; - if ((IsOpenCL || IsHIP) && Op != AtomicExpr::AO__opencl_atomic_init) + if ((IsOpenCL || IsHIP || IsScoped) && + Op != AtomicExpr::AO__opencl_atomic_init) ++AdjustedNumArgs; // Check we have the right number of arguments. if (Args.size() < AdjustedNumArgs) { diff --git a/clang/lib/Sema/SemaCodeComplete.cpp b/clang/lib/Sema/SemaCodeComplete.cpp index 6169144ef1c2..143968b4ab04 100644 --- a/clang/lib/Sema/SemaCodeComplete.cpp +++ b/clang/lib/Sema/SemaCodeComplete.cpp @@ -10112,7 +10112,7 @@ void Sema::CodeCompleteIncludedFile(llvm::StringRef Dir, bool Angled) { const StringRef &Dirname = llvm::sys::path::filename(Dir); const bool isQt = Dirname.startswith("Qt") || Dirname == "ActiveQt"; const bool ExtensionlessHeaders = - IsSystem || isQt || Dir.endswith(".framework/Headers"); + IsSystem || isQt || Dir.ends_with(".framework/Headers"); std::error_code EC; unsigned Count = 0; for (auto It = FS.dir_begin(Dir, EC); diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp index f12424d33b7d..19d972ed8ab2 100644 --- a/clang/lib/Sema/SemaDecl.cpp +++ b/clang/lib/Sema/SemaDecl.cpp @@ -5971,7 +5971,7 @@ Sema::GetNameFromUnqualifiedId(const UnqualifiedId &Name) { diag::err_deduction_guide_name_not_class_template) << (int)getTemplateNameKindForDiagnostics(TN) << TN; if (Template) - Diag(Template->getLocation(), diag::note_template_decl_here); + NoteTemplateLocation(*Template); return DeclarationNameInfo(); } @@ -8914,7 +8914,7 @@ void Sema::CheckVariableDeclarationType(VarDecl *NewVD) { } } - if (T->isRVVType()) + if (T->isRVVSizelessBuiltinType()) checkRVVTypeSupport(T, NewVD->getLocation(), cast(CurContext)); } @@ -16221,7 +16221,9 @@ Decl *Sema::ActOnFinishFunctionBody(Decl *dcl, Stmt *Body, ActivePolicy = &WP; } - if (!IsInstantiation && FD && FD->isConstexpr() && !FD->isInvalidDecl() && + if (!IsInstantiation && FD && + (FD->isConstexpr() || FD->hasAttr()) && + !FD->isInvalidDecl() && !CheckConstexprFunctionDefinition(FD, CheckConstexprKind::Diagnose)) FD->setInvalidDecl(); diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp index a345978bb870..59e456fd9f72 100644 --- a/clang/lib/Sema/SemaDeclAttr.cpp +++ b/clang/lib/Sema/SemaDeclAttr.cpp @@ -7372,6 +7372,28 @@ static void handleDeclspecThreadAttr(Sema &S, Decl *D, const ParsedAttr &AL) { D->addAttr(::new (S.Context) ThreadAttr(S.Context, AL)); } +static void handleMSConstexprAttr(Sema &S, Decl *D, const ParsedAttr &AL) { + if (!S.getLangOpts().isCompatibleWithMSVC(LangOptions::MSVC2022_3)) { + S.Diag(AL.getLoc(), diag::warn_unknown_attribute_ignored) + << AL << AL.getRange(); + return; + } + auto *FD = cast(D); + if (FD->isConstexprSpecified() || FD->isConsteval()) { + S.Diag(AL.getLoc(), diag::err_ms_constexpr_cannot_be_applied) + << FD->isConsteval() << FD; + return; + } + if (auto *MD = dyn_cast(FD)) { + if (!S.getLangOpts().CPlusPlus20 && MD->isVirtual()) { + S.Diag(AL.getLoc(), diag::err_ms_constexpr_cannot_be_applied) + << /*virtual*/ 2 << MD; + return; + } + } + D->addAttr(::new (S.Context) MSConstexprAttr(S.Context, AL)); +} + static void handleAbiTagAttr(Sema &S, Decl *D, const ParsedAttr &AL) { SmallVector Tags; for (unsigned I = 0, E = AL.getNumArgs(); I != E; ++I) { @@ -9477,6 +9499,9 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL, case ParsedAttr::AT_Thread: handleDeclspecThreadAttr(S, D, AL); break; + case ParsedAttr::AT_MSConstexpr: + handleMSConstexprAttr(S, D, AL); + break; // HLSL attributes: case ParsedAttr::AT_HLSLNumThreads: diff --git a/clang/lib/Sema/SemaDeclCXX.cpp b/clang/lib/Sema/SemaDeclCXX.cpp index 8fedf41d8424..36e53c684ac4 100644 --- a/clang/lib/Sema/SemaDeclCXX.cpp +++ b/clang/lib/Sema/SemaDeclCXX.cpp @@ -11462,7 +11462,7 @@ bool Sema::CheckDeductionGuideDeclarator(Declarator &D, QualType &R, GuidedTemplateDecl->getDeclContext()->getRedeclContext())) { Diag(D.getIdentifierLoc(), diag::err_deduction_guide_wrong_scope) << GuidedTemplateDecl; - Diag(GuidedTemplateDecl->getLocation(), diag::note_template_decl_here); + NoteTemplateLocation(*GuidedTemplateDecl); } auto &DS = D.getMutableDeclSpec(); @@ -17879,6 +17879,8 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, LookupResult Previous(*this, NameInfo, LookupOrdinaryName, ForExternalRedeclaration); + bool isTemplateId = D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId; + // There are five cases here. // - There's no scope specifier and we're in a local class. Only look // for functions declared in the immediately-enclosing block scope. @@ -17916,14 +17918,6 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, } adjustContextForLocalExternDecl(DC); - // C++ [class.friend]p6: - // A function can be defined in a friend declaration of a class if and - // only if the class is a non-local class (9.8), the function name is - // unqualified, and the function has namespace scope. - if (D.isFunctionDefinition()) { - Diag(NameInfo.getBeginLoc(), diag::err_friend_def_in_local_class); - } - // - There's no scope specifier, in which case we just go to the // appropriate scope and look for a function or function template // there as appropriate. @@ -17934,8 +17928,6 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, // elaborated-type-specifier, the lookup to determine whether // the entity has been previously declared shall not consider // any scopes outside the innermost enclosing namespace. - bool isTemplateId = - D.getName().getKind() == UnqualifiedIdKind::IK_TemplateId; // Find the appropriate context according to the above. DC = CurContext; @@ -17988,39 +17980,12 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, diag::warn_cxx98_compat_friend_is_member : diag::err_friend_is_member); - if (D.isFunctionDefinition()) { - // C++ [class.friend]p6: - // A function can be defined in a friend declaration of a class if and - // only if the class is a non-local class (9.8), the function name is - // unqualified, and the function has namespace scope. - // - // FIXME: We should only do this if the scope specifier names the - // innermost enclosing namespace; otherwise the fixit changes the - // meaning of the code. - SemaDiagnosticBuilder DB - = Diag(SS.getRange().getBegin(), diag::err_qualified_friend_def); - - DB << SS.getScopeRep(); - if (DC->isFileContext()) - DB << FixItHint::CreateRemoval(SS.getRange()); - SS.clear(); - } - // - There's a scope specifier that does not match any template // parameter lists, in which case we use some arbitrary context, // create a method or method template, and wait for instantiation. // - There's a scope specifier that does match some template // parameter lists, which we don't handle right now. } else { - if (D.isFunctionDefinition()) { - // C++ [class.friend]p6: - // A function can be defined in a friend declaration of a class if and - // only if the class is a non-local class (9.8), the function name is - // unqualified, and the function has namespace scope. - Diag(SS.getRange().getBegin(), diag::err_qualified_friend_def) - << SS.getScopeRep(); - } - DC = CurContext; assert(isa(DC) && "friend declaration not in class?"); } @@ -18105,6 +18070,38 @@ NamedDecl *Sema::ActOnFriendFunctionDecl(Scope *S, Declarator &D, else FD = cast(ND); + // C++ [class.friend]p6: + // A function may be defined in a friend declaration of a class if and + // only if the class is a non-local class, and the function name is + // unqualified. + if (D.isFunctionDefinition()) { + // Qualified friend function definition. + if (SS.isNotEmpty()) { + // FIXME: We should only do this if the scope specifier names the + // innermost enclosing namespace; otherwise the fixit changes the + // meaning of the code. + SemaDiagnosticBuilder DB = + Diag(SS.getRange().getBegin(), diag::err_qualified_friend_def); + + DB << SS.getScopeRep(); + if (DC->isFileContext()) + DB << FixItHint::CreateRemoval(SS.getRange()); + + // Friend function defined in a local class. + } else if (FunctionContainingLocalClass) { + Diag(NameInfo.getBeginLoc(), diag::err_friend_def_in_local_class); + + // Per [basic.pre]p4, a template-id is not a name. Therefore, if we have + // a template-id, the function name is not unqualified because these is + // no name. While the wording requires some reading in-between the + // lines, GCC, MSVC, and EDG all consider a friend function + // specialization definitions // to be de facto explicit specialization + // and diagnose them as such. + } else if (isTemplateId) { + Diag(NameInfo.getBeginLoc(), diag::err_friend_specialization_def); + } + } + // C++11 [dcl.fct.default]p4: If a friend declaration specifies a // default argument expression, that declaration shall be a definition // and shall be the only declaration of the function or function diff --git a/clang/lib/Sema/SemaInit.cpp b/clang/lib/Sema/SemaInit.cpp index 50ee0a5acb55..5ca6b232df66 100644 --- a/clang/lib/Sema/SemaInit.cpp +++ b/clang/lib/Sema/SemaInit.cpp @@ -4085,16 +4085,13 @@ static bool hasCopyOrMoveCtorParam(ASTContext &Ctx, return Ctx.hasSameUnqualifiedType(ParmT, ClassT); } -static OverloadingResult -ResolveConstructorOverload(Sema &S, SourceLocation DeclLoc, - MultiExprArg Args, - OverloadCandidateSet &CandidateSet, - QualType DestType, - DeclContext::lookup_result Ctors, - OverloadCandidateSet::iterator &Best, - bool CopyInitializing, bool AllowExplicit, - bool OnlyListConstructors, bool IsListInit, - bool SecondStepOfCopyInit = false) { +static OverloadingResult ResolveConstructorOverload( + Sema &S, SourceLocation DeclLoc, MultiExprArg Args, + OverloadCandidateSet &CandidateSet, QualType DestType, + DeclContext::lookup_result Ctors, OverloadCandidateSet::iterator &Best, + bool CopyInitializing, bool AllowExplicit, bool OnlyListConstructors, + bool IsListInit, bool RequireActualConstructor, + bool SecondStepOfCopyInit = false) { CandidateSet.clear(OverloadCandidateSet::CSK_InitByConstructor); CandidateSet.setDestAS(DestType.getQualifiers().getAddressSpace()); @@ -4157,7 +4154,7 @@ ResolveConstructorOverload(Sema &S, SourceLocation DeclLoc, // Note: SecondStepOfCopyInit is only ever true in this case when // evaluating whether to produce a C++98 compatibility warning. if (S.getLangOpts().CPlusPlus17 && Args.size() == 1 && - !SecondStepOfCopyInit) { + !RequireActualConstructor && !SecondStepOfCopyInit) { Expr *Initializer = Args[0]; auto *SourceRD = Initializer->getType()->getAsCXXRecordDecl(); if (SourceRD && S.isCompleteType(DeclLoc, Initializer->getType())) { @@ -4225,6 +4222,12 @@ static void TryConstructorInitialization(Sema &S, return; } + bool RequireActualConstructor = + !(Entity.getKind() != InitializedEntity::EK_Base && + Entity.getKind() != InitializedEntity::EK_Delegating && + Entity.getKind() != + InitializedEntity::EK_LambdaToBlockConversionBlockElement); + // C++17 [dcl.init]p17: // - If the initializer expression is a prvalue and the cv-unqualified // version of the source type is the same class as the class of the @@ -4234,11 +4237,7 @@ static void TryConstructorInitialization(Sema &S, // class or delegating to another constructor from a mem-initializer. // ObjC++: Lambda captured by the block in the lambda to block conversion // should avoid copy elision. - if (S.getLangOpts().CPlusPlus17 && - Entity.getKind() != InitializedEntity::EK_Base && - Entity.getKind() != InitializedEntity::EK_Delegating && - Entity.getKind() != - InitializedEntity::EK_LambdaToBlockConversionBlockElement && + if (S.getLangOpts().CPlusPlus17 && !RequireActualConstructor && UnwrappedArgs.size() == 1 && UnwrappedArgs[0]->isPRValue() && S.Context.hasSameUnqualifiedType(UnwrappedArgs[0]->getType(), DestType)) { // Convert qualifications if necessary. @@ -4286,11 +4285,10 @@ static void TryConstructorInitialization(Sema &S, // If the initializer list has no elements and T has a default constructor, // the first phase is omitted. if (!(UnwrappedArgs.empty() && S.LookupDefaultConstructor(DestRecordDecl))) - Result = ResolveConstructorOverload(S, Kind.getLocation(), Args, - CandidateSet, DestType, Ctors, Best, - CopyInitialization, AllowExplicit, - /*OnlyListConstructors=*/true, - IsListInit); + Result = ResolveConstructorOverload( + S, Kind.getLocation(), Args, CandidateSet, DestType, Ctors, Best, + CopyInitialization, AllowExplicit, + /*OnlyListConstructors=*/true, IsListInit, RequireActualConstructor); } // C++11 [over.match.list]p1: @@ -4300,11 +4298,10 @@ static void TryConstructorInitialization(Sema &S, // elements of the initializer list. if (Result == OR_No_Viable_Function) { AsInitializerList = false; - Result = ResolveConstructorOverload(S, Kind.getLocation(), UnwrappedArgs, - CandidateSet, DestType, Ctors, Best, - CopyInitialization, AllowExplicit, - /*OnlyListConstructors=*/false, - IsListInit); + Result = ResolveConstructorOverload( + S, Kind.getLocation(), UnwrappedArgs, CandidateSet, DestType, Ctors, + Best, CopyInitialization, AllowExplicit, + /*OnlyListConstructors=*/false, IsListInit, RequireActualConstructor); } if (Result) { Sequence.SetOverloadFailure( @@ -6778,6 +6775,7 @@ static ExprResult CopyObject(Sema &S, S, Loc, CurInitExpr, CandidateSet, T, Ctors, Best, /*CopyInitializing=*/false, /*AllowExplicit=*/true, /*OnlyListConstructors=*/false, /*IsListInit=*/false, + /*RequireActualConstructor=*/false, /*SecondStepOfCopyInit=*/true)) { case OR_Success: break; @@ -6920,6 +6918,7 @@ static void CheckCXX98CompatAccessibleCopy(Sema &S, S, Loc, CurInitExpr, CandidateSet, CurInitExpr->getType(), Ctors, Best, /*CopyInitializing=*/false, /*AllowExplicit=*/true, /*OnlyListConstructors=*/false, /*IsListInit=*/false, + /*RequireActualConstructor=*/false, /*SecondStepOfCopyInit=*/true); PartialDiagnostic Diag = S.PDiag(diag::warn_cxx98_compat_temp_copy) @@ -10573,7 +10572,7 @@ QualType Sema::DeduceTemplateSpecializationFromInitializer( diag::err_deduced_non_class_template_specialization_type) << (int)getTemplateNameKindForDiagnostics(TemplateName) << TemplateName; if (auto *TD = TemplateName.getAsTemplateDecl()) - Diag(TD->getLocation(), diag::note_template_decl_here); + NoteTemplateLocation(*TD); return QualType(); } diff --git a/clang/lib/Sema/SemaLambda.cpp b/clang/lib/Sema/SemaLambda.cpp index ca09b0481bca..4cc87c9fa765 100644 --- a/clang/lib/Sema/SemaLambda.cpp +++ b/clang/lib/Sema/SemaLambda.cpp @@ -1444,7 +1444,7 @@ void Sema::ActOnStartOfLambdaDefinition(LambdaIntroducer &Intro, for (const auto &Capture : Intro.Captures) { if (Capture.Id == TP->getIdentifier()) { Diag(Capture.Loc, diag::err_template_param_shadow) << Capture.Id; - Diag(TP->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*TP); } } } diff --git a/clang/lib/Sema/SemaRISCVVectorLookup.cpp b/clang/lib/Sema/SemaRISCVVectorLookup.cpp index 9a5aecf669a0..0d411fca0f9c 100644 --- a/clang/lib/Sema/SemaRISCVVectorLookup.cpp +++ b/clang/lib/Sema/SemaRISCVVectorLookup.cpp @@ -34,12 +34,6 @@ namespace { // Function definition of a RVV intrinsic. struct RVVIntrinsicDef { - /// Full function name with suffix, e.g. vadd_vv_i32m1. - std::string Name; - - /// Overloaded function name, e.g. vadd. - std::string OverloadName; - /// Mapping to which clang built-in function, e.g. __builtin_rvv_vadd. std::string BuiltinName; @@ -393,7 +387,7 @@ void RISCVIntrinsicManagerImpl::InitRVVIntrinsic( // Put into IntrinsicList. size_t Index = IntrinsicList.size(); - IntrinsicList.push_back({Name, OverloadedName, BuiltinName, Signature}); + IntrinsicList.push_back({BuiltinName, Signature}); // Creating mapping to Intrinsics. Intrinsics.insert({Name, Index}); diff --git a/clang/lib/Sema/SemaStmt.cpp b/clang/lib/Sema/SemaStmt.cpp index 2b45aa5dff7b..63348d27a8c9 100644 --- a/clang/lib/Sema/SemaStmt.cpp +++ b/clang/lib/Sema/SemaStmt.cpp @@ -1327,6 +1327,9 @@ Sema::ActOnFinishSwitchStmt(SourceLocation SwitchLoc, Stmt *Switch, } } + if (!TheDefaultStmt) + Diag(SwitchLoc, diag::warn_switch_default); + if (!HasDependentValue) { // If we don't have a default statement, check whether the // condition is constant. diff --git a/clang/lib/Sema/SemaStmtAttr.cpp b/clang/lib/Sema/SemaStmtAttr.cpp index eae1eaa2f956..725d8efe3828 100644 --- a/clang/lib/Sema/SemaStmtAttr.cpp +++ b/clang/lib/Sema/SemaStmtAttr.cpp @@ -397,6 +397,16 @@ static void CheckForDuplicateCodeAlignAttrs(Sema &S, } } +static Attr *handleMSConstexprAttr(Sema &S, Stmt *St, const ParsedAttr &A, + SourceRange Range) { + if (!S.getLangOpts().isCompatibleWithMSVC(LangOptions::MSVC2022_3)) { + S.Diag(A.getLoc(), diag::warn_unknown_attribute_ignored) + << A << A.getRange(); + return nullptr; + } + return ::new (S.Context) MSConstexprAttr(S.Context, A); +} + #define WANT_STMT_MERGE_LOGIC #include "clang/Sema/AttrParsedAttrImpl.inc" #undef WANT_STMT_MERGE_LOGIC @@ -600,6 +610,8 @@ static Attr *ProcessStmtAttribute(Sema &S, Stmt *St, const ParsedAttr &A, return handleUnlikely(S, St, A, Range); case ParsedAttr::AT_CodeAlign: return handleCodeAlignAttr(S, St, A); + case ParsedAttr::AT_MSConstexpr: + return handleMSConstexprAttr(S, St, A, Range); default: // N.B., ClangAttrEmitter.cpp emits a diagnostic helper that ensures a // declaration attribute is not written on a statement, but this code is diff --git a/clang/lib/Sema/SemaTemplate.cpp b/clang/lib/Sema/SemaTemplate.cpp index 09bbf14d39af..f10abeaba0d4 100644 --- a/clang/lib/Sema/SemaTemplate.cpp +++ b/clang/lib/Sema/SemaTemplate.cpp @@ -820,7 +820,6 @@ bool Sema::DiagnoseUninstantiableTemplate(SourceLocation PointOfInstantiation, if (!Complain || (PatternDef && PatternDef->isInvalidDecl())) return true; - std::optional Note; QualType InstantiationTy; if (TagDecl *TD = dyn_cast(Instantiation)) InstantiationTy = Context.getTypeDeclType(TD); @@ -838,25 +837,25 @@ bool Sema::DiagnoseUninstantiableTemplate(SourceLocation PointOfInstantiation, diag::err_explicit_instantiation_undefined_member) << /*member function*/ 1 << Instantiation->getDeclName() << Instantiation->getDeclContext(); - Note = diag::note_explicit_instantiation_here; + Diag(Pattern->getLocation(), diag::note_explicit_instantiation_here); } else { assert(isa(Instantiation) && "Must be a TagDecl!"); Diag(PointOfInstantiation, diag::err_implicit_instantiate_member_undefined) << InstantiationTy; - Note = diag::note_member_declared_at; + Diag(Pattern->getLocation(), diag::note_member_declared_at); } } else { if (isa(Instantiation)) { Diag(PointOfInstantiation, diag::err_explicit_instantiation_undefined_func_template) << Pattern; - Note = diag::note_explicit_instantiation_here; + Diag(Pattern->getLocation(), diag::note_explicit_instantiation_here); } else if (isa(Instantiation)) { Diag(PointOfInstantiation, diag::err_template_instantiate_undefined) << (TSK != TSK_ImplicitInstantiation) << InstantiationTy; - Note = diag::note_template_decl_here; + NoteTemplateLocation(*Pattern); } else { assert(isa(Instantiation) && "Must be a VarDecl!"); if (isa(Instantiation)) { @@ -869,11 +868,9 @@ bool Sema::DiagnoseUninstantiableTemplate(SourceLocation PointOfInstantiation, diag::err_explicit_instantiation_undefined_member) << /*static data member*/ 2 << Instantiation->getDeclName() << Instantiation->getDeclContext(); - Note = diag::note_explicit_instantiation_here; + Diag(Pattern->getLocation(), diag::note_explicit_instantiation_here); } } - if (Note) // Diagnostics were emitted. - Diag(Pattern->getLocation(), *Note); // In general, Instantiation isn't marked invalid to get more than one // error for multiple undefined instantiations. But the code that does @@ -898,8 +895,9 @@ void Sema::DiagnoseTemplateParameterShadow(SourceLocation Loc, Decl *PrevDecl) { // Make this a warning when MSVC compatibility is requested. unsigned DiagId = getLangOpts().MSVCCompat ? diag::ext_template_param_shadow : diag::err_template_param_shadow; - Diag(Loc, DiagId) << cast(PrevDecl)->getDeclName(); - Diag(PrevDecl->getLocation(), diag::note_template_param_here); + const auto *ND = cast(PrevDecl); + Diag(Loc, DiagId) << ND->getDeclName(); + NoteTemplateParameterLocation(*ND); } /// AdjustDeclIfTemplate - If the given decl happens to be a template, reset @@ -1714,6 +1712,8 @@ class ConstraintRefersToContainingTemplateChecker // Friend, likely because it was referred to without its template arguments. void CheckIfContainingRecord(const CXXRecordDecl *CheckingRD) { CheckingRD = CheckingRD->getMostRecentDecl(); + if (!CheckingRD->isTemplated()) + return; for (const DeclContext *DC = Friend->getLexicalDeclContext(); DC && !DC->isFileContext(); DC = DC->getParent()) @@ -4488,7 +4488,7 @@ static void checkMoreSpecializedThanPrimary(Sema &S, PartialSpecDecl *Partial) { << SFINAEArgString; } - S.Diag(Template->getLocation(), diag::note_template_decl_here); + S.NoteTemplateLocation(*Template); SmallVector PartialAC, TemplateAC; Template->getAssociatedConstraints(TemplateAC); Partial->getAssociatedConstraints(PartialAC); @@ -4966,8 +4966,7 @@ void Sema::diagnoseMissingTemplateArguments(TemplateName Name, Diag(Loc, diag::err_template_missing_args) << (int)getTemplateNameKindForDiagnostics(Name) << Name; if (TemplateDecl *TD = Name.getAsTemplateDecl()) { - Diag(TD->getLocation(), diag::note_template_decl_here) - << TD->getTemplateParameters()->getSourceRange(); + NoteTemplateLocation(*TD, TD->getTemplateParameters()->getSourceRange()); } } @@ -5319,7 +5318,7 @@ bool Sema::CheckTemplateTypeArgument( ? diag::ext_ms_template_type_arg_missing_typename : diag::err_template_arg_must_be_type_suggest) << FixItHint::CreateInsertion(Loc, "typename "); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); // Recover by synthesizing a type using the location information that we // already have. @@ -5348,7 +5347,7 @@ bool Sema::CheckTemplateTypeArgument( // is not a type. SourceRange SR = AL.getSourceRange(); Diag(SR.getBegin(), diag::err_template_arg_must_be_type) << SR; - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return true; } @@ -5794,8 +5793,8 @@ bool Sema::CheckTemplateArgument( // therefore cannot be a non-type template argument. Diag(Arg.getLocation(), diag::err_template_arg_must_be_expr) << Arg.getSourceRange(); + NoteTemplateParameterLocation(*Param); - Diag(Param->getLocation(), diag::note_template_param_here); return true; case TemplateArgument::Type: { @@ -5815,7 +5814,7 @@ bool Sema::CheckTemplateArgument( Diag(SR.getBegin(), diag::err_template_arg_nontype_ambig) << SR << T; else Diag(SR.getBegin(), diag::err_template_arg_must_be_expr) << SR; - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return true; } @@ -5936,8 +5935,7 @@ static bool diagnoseMissingArgument(Sema &S, SourceLocation Loc, << /*not enough args*/0 << (int)S.getTemplateNameKindForDiagnostics(TemplateName(TD)) << TD; - S.Diag(TD->getLocation(), diag::note_template_decl_here) - << Params->getSourceRange(); + S.NoteTemplateLocation(*TD, Params->getSourceRange()); return true; } @@ -6003,8 +6001,7 @@ bool Sema::CheckTemplateArgumentList( << /*not enough args*/0 << (int)getTemplateNameKindForDiagnostics(TemplateName(Template)) << Template; - Diag(Template->getLocation(), diag::note_template_decl_here) - << Params->getSourceRange(); + NoteTemplateLocation(*Template, Params->getSourceRange()); return true; } } @@ -6036,7 +6033,7 @@ bool Sema::CheckTemplateArgumentList( diag::err_template_expansion_into_fixed_list) << (isa(Template) ? 1 : 0) << NewArgs[ArgIdx].getSourceRange(); - Diag((*Param)->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(**Param); return true; } @@ -6232,8 +6229,7 @@ bool Sema::CheckTemplateArgumentList( << (int)getTemplateNameKindForDiagnostics(TemplateName(Template)) << Template << SourceRange(NewArgs[ArgIdx].getLocation(), NewArgs.getRAngleLoc()); - Diag(Template->getLocation(), diag::note_template_decl_here) - << Params->getSourceRange(); + NoteTemplateLocation(*Template, Params->getSourceRange()); return true; } @@ -6645,7 +6641,7 @@ isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param, for (unsigned I = 0, N = Notes.size(); I != N; ++I) S.Diag(Notes[I].first, Notes[I].second); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return NPV_Error; } @@ -6671,7 +6667,7 @@ isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param, // then recover as if the types were correct. S.Diag(Arg->getExprLoc(), diag::err_template_arg_wrongtype_null_constant) << Arg->getType() << ParamType << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return NPV_NullPointer; } @@ -6681,7 +6677,7 @@ isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param, // message with the information we have here. S.Diag(Arg->getExprLoc(), diag::err_template_arg_invalid) << EvalResult.Val.getAsString(S.Context, ParamType); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return NPV_Error; } @@ -6693,7 +6689,7 @@ isNullPointerValueTemplateArgument(Sema &S, NonTypeTemplateParmDecl *Param, << ParamType << FixItHint::CreateInsertion(Arg->getBeginLoc(), Code) << FixItHint::CreateInsertion(S.getLocForEndOfToken(Arg->getEndLoc()), ")"); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return NPV_NullPointer; } @@ -6734,7 +6730,7 @@ static bool CheckTemplateArgumentIsCompatibleWithParameter( S.Diag(Arg->getBeginLoc(), diag::err_template_arg_ref_bind_ignores_quals) << ParamType << Arg->getType() << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } } @@ -6752,7 +6748,7 @@ static bool CheckTemplateArgumentIsCompatibleWithParameter( else S.Diag(Arg->getBeginLoc(), diag::err_template_arg_not_convertible) << ArgIn->getType() << ParamType << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } } @@ -6895,7 +6891,7 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction( if (!Entity) { S.Diag(Arg->getBeginLoc(), diag::err_template_arg_not_decl_ref) << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } @@ -6903,7 +6899,7 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction( if (isa(Entity) || isa(Entity)) { S.Diag(Arg->getBeginLoc(), diag::err_template_arg_field) << Entity << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } @@ -6912,7 +6908,7 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction( if (!Method->isStatic()) { S.Diag(Arg->getBeginLoc(), diag::err_template_arg_method) << Method << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } } @@ -6952,7 +6948,7 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction( if (Var->getType()->isReferenceType()) { S.Diag(Arg->getBeginLoc(), diag::err_template_arg_reference_var) << Var->getType() << Arg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } @@ -6973,14 +6969,14 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction( ParamType.getNonReferenceType())) { S.Diag(AddrOpLoc, diag::err_template_arg_address_of_non_pointer) << ParamType; - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } S.Diag(AddrOpLoc, diag::err_template_arg_address_of_non_pointer) << ParamType << FixItHint::CreateRemoval(AddrOpLoc); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); ArgType = Entity->getType(); } @@ -7002,14 +6998,14 @@ static bool CheckTemplateArgumentAddressOfObjectOrFunction( if (!S.Context.hasSameUnqualifiedType(ArgType, ParamType)) { S.Diag(Arg->getBeginLoc(), diag::err_template_arg_not_address_of) << ParamType; - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } S.Diag(Arg->getBeginLoc(), diag::err_template_arg_not_address_of) << ParamType << FixItHint::CreateInsertion(Arg->getBeginLoc(), "&"); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); } } @@ -7125,7 +7121,7 @@ CheckTemplateArgumentPointerToMember(Sema &S, NonTypeTemplateParmDecl *Param, // We can't perform this conversion. S.Diag(ResultArg->getBeginLoc(), diag::err_template_arg_not_convertible) << ResultArg->getType() << ParamType << ResultArg->getSourceRange(); - S.Diag(Param->getLocation(), diag::note_template_param_here); + S.NoteTemplateParameterLocation(*Param); return true; } @@ -7237,7 +7233,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, diag::err_non_type_template_parm_type_deduction_failure) << Param->getDeclName() << Param->getType() << Arg->getType() << Arg->getSourceRange(); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return ExprError(); } } @@ -7247,7 +7243,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, // where the parameter type is deduced. ParamType = CheckNonTypeTemplateParameterType(ParamType, Arg->getExprLoc()); if (ParamType.isNull()) { - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return ExprError(); } } @@ -7283,7 +7279,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, Diag(StartLoc, diag::err_deduced_non_type_template_arg_type_mismatch) << Arg->getType() << ParamType.getUnqualifiedType(); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return ExprError(); } @@ -7580,7 +7576,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, if (!ArgType->isIntegralOrEnumerationType()) { Diag(Arg->getBeginLoc(), diag::err_template_arg_not_integral_or_enumeral) << ArgType << Arg->getSourceRange(); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return ExprError(); } else if (!Arg->isValueDependent()) { class TmplArgICEDiagnoser : public VerifyICEDiagnoser { @@ -7618,7 +7614,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, // We can't perform this conversion. Diag(Arg->getBeginLoc(), diag::err_template_arg_not_convertible) << Arg->getType() << ParamType << Arg->getSourceRange(); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return ExprError(); } @@ -7664,7 +7660,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, Diag(Arg->getBeginLoc(), diag::warn_template_arg_negative) << toString(OldValue, 10) << toString(Value, 10) << Param->getType() << Arg->getSourceRange(); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); } // Complain if we overflowed the template parameter's type. @@ -7679,7 +7675,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, Diag(Arg->getBeginLoc(), diag::warn_template_arg_too_large) << toString(OldValue, 10) << toString(Value, 10) << Param->getType() << Arg->getSourceRange(); - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); } } @@ -7806,7 +7802,7 @@ ExprResult Sema::CheckTemplateArgument(NonTypeTemplateParmDecl *Param, case NPV_NotNullPointer: Diag(Arg->getExprLoc(), diag::err_template_arg_not_convertible) << Arg->getType() << ParamType; - Diag(Param->getLocation(), diag::note_template_param_here); + NoteTemplateParameterLocation(*Param); return ExprError(); case NPV_Error: @@ -7941,6 +7937,37 @@ bool Sema::CheckTemplateTemplateArgument(TemplateTemplateParmDecl *Param, Arg.getLocation()); } +static Sema::SemaDiagnosticBuilder noteLocation(Sema &S, const NamedDecl &Decl, + unsigned HereDiagID, + unsigned ExternalDiagID) { + if (Decl.getLocation().isValid()) + return S.Diag(Decl.getLocation(), HereDiagID); + + SmallString<128> Str; + llvm::raw_svector_ostream Out(Str); + PrintingPolicy PP = S.getPrintingPolicy(); + PP.TerseOutput = 1; + Decl.print(Out, PP); + return S.Diag(Decl.getLocation(), ExternalDiagID) << Out.str(); +} + +void Sema::NoteTemplateLocation(const NamedDecl &Decl, + std::optional ParamRange) { + SemaDiagnosticBuilder DB = + noteLocation(*this, Decl, diag::note_template_decl_here, + diag::note_template_decl_external); + if (ParamRange && ParamRange->isValid()) { + assert(Decl.getLocation().isValid() && + "Parameter range has location when Decl does not"); + DB << *ParamRange; + } +} + +void Sema::NoteTemplateParameterLocation(const NamedDecl &Decl) { + noteLocation(*this, Decl, diag::note_template_param_here, + diag::note_template_param_external); +} + /// Given a non-type template argument that refers to a /// declaration and the type of its corresponding non-type template /// parameter, produce an expression that properly refers to that @@ -8666,9 +8693,7 @@ static bool CheckNonTypeTemplatePartialSpecializationArgs( S.Diag(IsDefaultArgument ? TemplateNameLoc : ArgExpr->getBeginLoc(), diag::err_dependent_typed_non_type_arg_in_partial_spec) << Param->getType(); - S.Diag(Param->getLocation(), diag::note_template_param_here) - << (IsDefaultArgument ? ParamUseRange : SourceRange()) - << ParamUseRange; + S.NoteTemplateParameterLocation(*Param); return true; } } @@ -11248,7 +11273,7 @@ Sema::CheckTypenameType(ElaboratedTypeKeyword Keyword, else Diag(IILoc, diag::err_deduced_tst) << (int)getTemplateNameKindForDiagnostics(TemplateName(TD)); - Diag(TD->getLocation(), diag::note_template_decl_here); + NoteTemplateLocation(*TD); return QualType(); } return Context.getElaboratedType( diff --git a/clang/lib/Sema/SemaTemplateInstantiate.cpp b/clang/lib/Sema/SemaTemplateInstantiate.cpp index 4ae027bd1bb6..df6b40999e64 100644 --- a/clang/lib/Sema/SemaTemplateInstantiate.cpp +++ b/clang/lib/Sema/SemaTemplateInstantiate.cpp @@ -810,6 +810,10 @@ void Sema::PrintInstantiationStack() { Diags.Report(Active->PointOfInstantiation, diag::note_template_nsdmi_here) << FD << Active->InstantiationRange; + } else if (ClassTemplateDecl *CTD = dyn_cast(D)) { + Diags.Report(Active->PointOfInstantiation, + diag::note_template_class_instantiation_here) + << CTD << Active->InstantiationRange; } else { Diags.Report(Active->PointOfInstantiation, diag::note_template_type_alias_instantiation_here) diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp index 56d133f20a29..83610503ed9b 100644 --- a/clang/lib/Sema/SemaType.cpp +++ b/clang/lib/Sema/SemaType.cpp @@ -3792,7 +3792,7 @@ static QualType GetDeclSpecTypeForDeclarator(TypeProcessingState &state, << Kind << Error << (int)SemaRef.getTemplateNameKindForDiagnostics(TN) << QualType(Deduced, 0) << AutoRange; if (auto *TD = TN.getAsTemplateDecl()) - SemaRef.Diag(TD->getLocation(), diag::note_template_decl_here); + SemaRef.NoteTemplateLocation(*TD); T = SemaRef.Context.IntTy; D.setInvalidType(true); diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp index f22da838424b..5b51ac40000d 100644 --- a/clang/lib/Serialization/ASTReader.cpp +++ b/clang/lib/Serialization/ASTReader.cpp @@ -2531,8 +2531,7 @@ InputFile ASTReader::getInputFile(ModuleFile &F, unsigned ID, bool Complain) { Overridden = false; } - OptionalFileEntryRefDegradesToFileEntryPtr File = OptionalFileEntryRef( - expectedToOptional(FileMgr.getFileRef(Filename, /*OpenFile=*/false))); + auto File = FileMgr.getOptionalFileRef(Filename, /*OpenFile=*/false); // For an overridden file, create a virtual file with the stored // size/timestamp. @@ -2559,7 +2558,8 @@ InputFile ASTReader::getInputFile(ModuleFile &F, unsigned ID, bool Complain) { // PCH. SourceManager &SM = getSourceManager(); // FIXME: Reject if the overrides are different. - if ((!Overridden && !Transient) && !SkipChecks && SM.isFileOverridden(File)) { + if ((!Overridden && !Transient) && !SkipChecks && + SM.isFileOverridden(*File)) { if (Complain) Error(diag::err_fe_pch_file_overridden, Filename); @@ -3152,7 +3152,7 @@ ASTReader::ReadControlBlock(ModuleFile &F, if (!bool(PP.getPreprocessorOpts().DisablePCHOrModuleValidation & DisableValidationForModuleKind::Module) && F.Kind != MK_ExplicitModule && F.Kind != MK_PrebuiltModule) { - auto BuildDir = PP.getFileManager().getDirectory(Blob); + auto BuildDir = PP.getFileManager().getOptionalDirectoryRef(Blob); if (!BuildDir || *BuildDir != M->Directory) { if (!canRecoverFromOutOfDate(F.FileName, ClientLoadCapabilities)) Diag(diag::err_imported_module_relocated) @@ -5786,7 +5786,7 @@ llvm::Error ASTReader::ReadSubmoduleBlock(ModuleFile &F, PartialDiagnostic(diag::err_module_file_conflict, ContextObj->DiagAllocator) << CurrentModule->getTopLevelModuleName() << CurFile->getName() - << F.File->getName(); + << F.File.getName(); return DiagnosticError::create(CurrentImportLoc, ConflictError); } } diff --git a/clang/lib/Serialization/ASTReaderDecl.cpp b/clang/lib/Serialization/ASTReaderDecl.cpp index bc16cfc67a24..7140a14aefbf 100644 --- a/clang/lib/Serialization/ASTReaderDecl.cpp +++ b/clang/lib/Serialization/ASTReaderDecl.cpp @@ -583,6 +583,9 @@ void ASTDeclReader::Visit(Decl *D) { } void ASTDeclReader::VisitDecl(Decl *D) { + BitsUnpacker DeclBits(Record.readInt()); + bool HasStandaloneLexicalDC = DeclBits.getNextBit(); + if (D->isTemplateParameter() || D->isTemplateParameterPack() || isa(D)) { // We don't want to deserialize the DeclContext of a template @@ -592,7 +595,8 @@ void ASTDeclReader::VisitDecl(Decl *D) { // return type of the function). Use the translation unit DeclContext as a // placeholder. GlobalDeclID SemaDCIDForTemplateParmDecl = readDeclID(); - GlobalDeclID LexicalDCIDForTemplateParmDecl = readDeclID(); + GlobalDeclID LexicalDCIDForTemplateParmDecl = + HasStandaloneLexicalDC ? readDeclID() : 0; if (!LexicalDCIDForTemplateParmDecl) LexicalDCIDForTemplateParmDecl = SemaDCIDForTemplateParmDecl; Reader.addPendingDeclContextInfo(D, @@ -601,7 +605,8 @@ void ASTDeclReader::VisitDecl(Decl *D) { D->setDeclContext(Reader.getContext().getTranslationUnitDecl()); } else { auto *SemaDC = readDeclAs(); - auto *LexicalDC = readDeclAs(); + auto *LexicalDC = + HasStandaloneLexicalDC ? readDeclAs() : nullptr; if (!LexicalDC) LexicalDC = SemaDC; // If the context is a class, we might not have actually merged it yet, in @@ -618,7 +623,6 @@ void ASTDeclReader::VisitDecl(Decl *D) { } D->setLocation(ThisDeclLoc); - BitsUnpacker DeclBits(Record.readInt()); D->InvalidDecl = DeclBits.getNextBit(); bool HasAttrs = DeclBits.getNextBit(); D->setImplicit(DeclBits.getNextBit()); @@ -765,7 +769,7 @@ ASTDeclReader::RedeclarableResult ASTDeclReader::VisitTagDecl(TagDecl *TD) { TD->setCompleteDefinitionRequired(TagDeclBits.getNextBit()); TD->setBraceRange(readSourceRange()); - switch (Record.readInt()) { + switch (TagDeclBits.getNextBits(/*Width=*/2)) { case 0: break; case 1: { // ExtInfo @@ -1089,7 +1093,8 @@ void ASTDeclReader::VisitFunctionDecl(FunctionDecl *FD) { FD->setCachedLinkage((Linkage)FunctionDeclBits.getNextBits(/*Width=*/3)); FD->EndRangeLoc = readSourceLocation(); - FD->setDefaultLoc(readSourceLocation()); + if (FD->isExplicitlyDefaulted()) + FD->setDefaultLoc(readSourceLocation()); FD->ODRHash = Record.readInt(); FD->setHasODRHash(true); @@ -1703,7 +1708,7 @@ void ASTDeclReader::VisitParmVarDecl(ParmVarDecl *PD) { unsigned isObjCMethodParam = ParmVarDeclBits.getNextBit(); unsigned scopeDepth = ParmVarDeclBits.getNextBits(/*Width=*/7); unsigned scopeIndex = ParmVarDeclBits.getNextBits(/*Width=*/8); - unsigned declQualifier = Record.readInt(); + unsigned declQualifier = ParmVarDeclBits.getNextBits(/*Width=*/7); if (isObjCMethodParam) { assert(scopeDepth == 0); PD->setObjCMethodScopeInfo(scopeIndex); @@ -1716,7 +1721,9 @@ void ASTDeclReader::VisitParmVarDecl(ParmVarDecl *PD) { PD->ParmVarDeclBits.HasInheritedDefaultArg = ParmVarDeclBits.getNextBit(); if (ParmVarDeclBits.getNextBit()) // hasUninstantiatedDefaultArg. PD->setUninstantiatedDefaultArg(Record.readExpr()); - PD->ExplicitObjectParameterIntroducerLoc = Record.readSourceLocation(); + + if (ParmVarDeclBits.getNextBit()) // Valid explicit object parameter + PD->ExplicitObjectParameterIntroducerLoc = Record.readSourceLocation(); // FIXME: If this is a redeclaration of a function from another module, handle // inheritance of default arguments. diff --git a/clang/lib/Serialization/ASTReaderStmt.cpp b/clang/lib/Serialization/ASTReaderStmt.cpp index d7d0c0e5bb21..b3a6f619372b 100644 --- a/clang/lib/Serialization/ASTReaderStmt.cpp +++ b/clang/lib/Serialization/ASTReaderStmt.cpp @@ -108,7 +108,7 @@ namespace clang { /// The number of record fields required for the Expr class /// itself. - static const unsigned NumExprFields = NumStmtFields + 4; + static const unsigned NumExprFields = NumStmtFields + 2; /// Read and initialize a ExplicitTemplateArgumentList structure. void ReadTemplateKWAndArgsInfo(ASTTemplateKWAndArgsInfo &Args, @@ -524,9 +524,13 @@ void ASTStmtReader::VisitCapturedStmt(CapturedStmt *S) { void ASTStmtReader::VisitExpr(Expr *E) { VisitStmt(E); E->setType(Record.readType()); - E->setDependence(static_cast(Record.readInt())); - E->setValueKind(static_cast(Record.readInt())); - E->setObjectKind(static_cast(Record.readInt())); + BitsUnpacker ExprBits(Record.readInt()); + E->setDependence( + static_cast(ExprBits.getNextBits(/*Width=*/5))); + E->setValueKind( + static_cast(ExprBits.getNextBits(/*Width=*/2))); + E->setObjectKind( + static_cast(ExprBits.getNextBits(/*Width=*/3))); assert(Record.getIdx() == NumExprFields && "Incorrect expression field count"); } @@ -995,14 +999,19 @@ void ASTStmtReader::VisitOMPIteratorExpr(OMPIteratorExpr *E) { void ASTStmtReader::VisitCallExpr(CallExpr *E) { VisitExpr(E); - unsigned NumArgs = Record.readInt(); - bool HasFPFeatures = Record.readInt(); + + BitsUnpacker CallExprBits = Record.readInt(); + + unsigned NumArgs = CallExprBits.getNextBits(/*Width=*/16); + bool HasFPFeatures = CallExprBits.getNextBit(); + E->setADLCallKind( + static_cast(CallExprBits.getNextBit())); assert((NumArgs == E->getNumArgs()) && "Wrong NumArgs!"); E->setRParenLoc(readSourceLocation()); E->setCallee(Record.readSubExpr()); for (unsigned I = 0; I != NumArgs; ++I) E->setArg(I, Record.readSubExpr()); - E->setADLCallKind(static_cast(Record.readInt())); + if (HasFPFeatures) E->setStoredFPFeatures( FPOptionsOverride::getFromOpaqueInt(Record.readInt())); @@ -2013,14 +2022,15 @@ ASTStmtReader::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) { void ASTStmtReader::VisitOverloadExpr(OverloadExpr *E) { VisitExpr(E); - unsigned NumResults = Record.readInt(); - bool HasTemplateKWAndArgsInfo = Record.readInt(); + BitsUnpacker OverloadExprBits = Record.readInt(); + unsigned NumResults = OverloadExprBits.getNextBits(/*Width=*/14); + bool HasTemplateKWAndArgsInfo = OverloadExprBits.getNextBit(); assert((E->getNumDecls() == NumResults) && "Wrong NumResults!"); assert((E->hasTemplateKWAndArgsInfo() == HasTemplateKWAndArgsInfo) && "Wrong HasTemplateKWAndArgsInfo!"); if (HasTemplateKWAndArgsInfo) { - unsigned NumTemplateArgs = Record.readInt(); + unsigned NumTemplateArgs = OverloadExprBits.getNextBits(/*Width=*/14); ReadTemplateKWAndArgsInfo(*E->getTrailingASTTemplateKWAndArgsInfo(), E->getTrailingTemplateArgumentLoc(), NumTemplateArgs); @@ -3022,11 +3032,13 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { Record[ASTStmtReader::NumExprFields]); break; - case EXPR_CALL: - S = CallExpr::CreateEmpty( - Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty); + case EXPR_CALL: { + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto HasFPFeatures = CallExprBits.getNextBit(); + S = CallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, Empty); break; + } case EXPR_RECOVERY: S = RecoveryExpr::CreateEmpty( @@ -3764,17 +3776,23 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { break; } - case EXPR_CXX_OPERATOR_CALL: - S = CXXOperatorCallExpr::CreateEmpty( - Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty); + case EXPR_CXX_OPERATOR_CALL: { + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto HasFPFeatures = CallExprBits.getNextBit(); + S = CXXOperatorCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, + Empty); break; + } - case EXPR_CXX_MEMBER_CALL: - S = CXXMemberCallExpr::CreateEmpty( - Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty); + case EXPR_CXX_MEMBER_CALL: { + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto HasFPFeatures = CallExprBits.getNextBit(); + S = CXXMemberCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, + Empty); break; + } case EXPR_CXX_REWRITTEN_BINARY_OPERATOR: S = new (Context) CXXRewrittenBinaryOperator(Empty); @@ -3833,11 +3851,14 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { S = new (Context) BuiltinBitCastExpr(Empty); break; - case EXPR_USER_DEFINED_LITERAL: - S = UserDefinedLiteral::CreateEmpty( - Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty); + case EXPR_USER_DEFINED_LITERAL: { + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto HasFPFeatures = CallExprBits.getNextBit(); + S = UserDefinedLiteral::CreateEmpty(Context, NumArgs, HasFPFeatures, + Empty); break; + } case EXPR_CXX_STD_INITIALIZER_LIST: S = new (Context) CXXStdInitializerListExpr(Empty); @@ -3948,23 +3969,21 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { case EXPR_CXX_UNRESOLVED_MEMBER: S = UnresolvedMemberExpr::CreateEmpty( Context, - /*NumResults=*/Record[ASTStmtReader::NumExprFields], - /*HasTemplateKWAndArgsInfo=*/Record[ASTStmtReader::NumExprFields + 1], - /*NumTemplateArgs=*/ - Record[ASTStmtReader::NumExprFields + 1] - ? Record[ASTStmtReader::NumExprFields + 2] - : 0); + /*NumResults=*/Record[ASTStmtReader::NumExprFields] & ((1 << 14) - 1), + /*HasTemplateKWAndArgsInfo=*/ + (Record[ASTStmtReader::NumExprFields] >> 14) & (0x1), + /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields] >> 14 & + ((1 << 14) - 1)); break; case EXPR_CXX_UNRESOLVED_LOOKUP: S = UnresolvedLookupExpr::CreateEmpty( Context, - /*NumResults=*/Record[ASTStmtReader::NumExprFields], - /*HasTemplateKWAndArgsInfo=*/Record[ASTStmtReader::NumExprFields + 1], - /*NumTemplateArgs=*/ - Record[ASTStmtReader::NumExprFields + 1] - ? Record[ASTStmtReader::NumExprFields + 2] - : 0); + /*NumResults=*/Record[ASTStmtReader::NumExprFields] & ((1 << 14) - 1), + /*HasTemplateKWAndArgsInfo=*/ + (Record[ASTStmtReader::NumExprFields] >> 14) & (0x1), + /*NumTemplateArgs=*/Record[ASTStmtReader::NumExprFields] >> 14 & + ((1 << 14) - 1)); break; case EXPR_TYPE_TRAIT: @@ -4024,11 +4043,14 @@ Stmt *ASTReader::ReadStmtFromStream(ModuleFile &F) { S = new (Context) OpaqueValueExpr(Empty); break; - case EXPR_CUDA_KERNEL_CALL: - S = CUDAKernelCallExpr::CreateEmpty( - Context, /*NumArgs=*/Record[ASTStmtReader::NumExprFields], - /*HasFPFeatures=*/Record[ASTStmtReader::NumExprFields + 1], Empty); + case EXPR_CUDA_KERNEL_CALL: { + BitsUnpacker CallExprBits(Record[ASTStmtReader::NumExprFields]); + auto NumArgs = CallExprBits.getNextBits(/*Width=*/16); + auto HasFPFeatures = CallExprBits.getNextBit(); + S = CUDAKernelCallExpr::CreateEmpty(Context, NumArgs, HasFPFeatures, + Empty); break; + } case EXPR_ASTYPE: S = new (Context) AsTypeExpr(Empty); diff --git a/clang/lib/Serialization/ASTWriter.cpp b/clang/lib/Serialization/ASTWriter.cpp index 6df815234e23..91eb2af8f8ad 100644 --- a/clang/lib/Serialization/ASTWriter.cpp +++ b/clang/lib/Serialization/ASTWriter.cpp @@ -1413,7 +1413,7 @@ void ASTWriter::WriteControlBlock(Preprocessor &PP, ASTContext &Context, // If we have calculated signature, there is no need to store // the size or timestamp. - Record.push_back(M.Signature ? 0 : M.File->getSize()); + Record.push_back(M.Signature ? 0 : M.File.getSize()); Record.push_back(M.Signature ? 0 : getTimestampForOutput(M.File)); llvm::append_range(Record, M.Signature); @@ -2182,8 +2182,8 @@ void ASTWriter::WriteSourceManagerBlock(SourceManager &SourceMgr, "Writing to AST an overridden file is not supported"); // The source location entry is a file. Emit input file ID. - assert(InputFileIDs[Content->OrigEntry] != 0 && "Missed file entry"); - Record.push_back(InputFileIDs[Content->OrigEntry]); + assert(InputFileIDs[*Content->OrigEntry] != 0 && "Missed file entry"); + Record.push_back(InputFileIDs[*Content->OrigEntry]); Record.push_back(getAdjustedNumCreatedFIDs(FID)); @@ -4695,7 +4695,7 @@ void ASTWriter::collectNonAffectingInputFiles() { if (!isModuleMap(File.getFileCharacteristic()) || AffectingModuleMaps.empty() || - AffectingModuleMaps.find(Cache->OrigEntry) != AffectingModuleMaps.end()) + llvm::is_contained(AffectingModuleMaps, *Cache->OrigEntry)) continue; IsSLocAffecting[I] = false; diff --git a/clang/lib/Serialization/ASTWriterDecl.cpp b/clang/lib/Serialization/ASTWriterDecl.cpp index bf082e5b8eac..43169b2befc6 100644 --- a/clang/lib/Serialization/ASTWriterDecl.cpp +++ b/clang/lib/Serialization/ASTWriterDecl.cpp @@ -320,13 +320,8 @@ void ASTDeclWriter::Visit(Decl *D) { } void ASTDeclWriter::VisitDecl(Decl *D) { - Record.AddDeclRef(cast_or_null(D->getDeclContext())); - if (D->getDeclContext() != D->getLexicalDeclContext()) - Record.AddDeclRef(cast_or_null(D->getLexicalDeclContext())); - else - Record.push_back(0); - BitsPacker DeclBits; + DeclBits.addBit(D->getDeclContext() != D->getLexicalDeclContext()); DeclBits.addBit(D->isInvalidDecl()); DeclBits.addBit(D->hasAttrs()); DeclBits.addBit(D->isImplicit()); @@ -337,6 +332,10 @@ void ASTDeclWriter::VisitDecl(Decl *D) { DeclBits.addBits((uint64_t)D->getModuleOwnershipKind(), /*BitWidth=*/3); Record.push_back(DeclBits); + Record.AddDeclRef(cast_or_null(D->getDeclContext())); + if (D->getDeclContext() != D->getLexicalDeclContext()) + Record.AddDeclRef(cast_or_null(D->getLexicalDeclContext())); + if (D->hasAttrs()) Record.AddAttributes(D->getAttrs()); @@ -450,19 +449,18 @@ void ASTDeclWriter::VisitTagDecl(TagDecl *D) { TagDeclBits.addBit(D->isEmbeddedInDeclarator()); TagDeclBits.addBit(D->isFreeStanding()); TagDeclBits.addBit(D->isCompleteDefinitionRequired()); + TagDeclBits.addBits( + D->hasExtInfo() ? 1 : (D->getTypedefNameForAnonDecl() ? 2 : 0), + /*BitWidth=*/2); Record.push_back(TagDeclBits); Record.AddSourceRange(D->getBraceRange()); if (D->hasExtInfo()) { - Record.push_back(1); Record.AddQualifierInfo(*D->getExtInfo()); } else if (auto *TD = D->getTypedefNameForAnonDecl()) { - Record.push_back(2); Record.AddDeclRef(TD); Record.AddIdentifierRef(TD->getDeclName().getAsIdentifierInfo()); - } else { - Record.push_back(0); } } @@ -702,7 +700,8 @@ void ASTDeclWriter::VisitFunctionDecl(FunctionDecl *D) { Record.push_back(FunctionDeclBits); Record.AddSourceLocation(D->getEndLoc()); - Record.AddSourceLocation(D->getDefaultLoc()); + if (D->isExplicitlyDefaulted()) + Record.AddSourceLocation(D->getDefaultLoc()); Record.push_back(D->getODRHash()); @@ -1176,15 +1175,18 @@ void ASTDeclWriter::VisitParmVarDecl(ParmVarDecl *D) { ParmVarDeclBits.addBit(D->isObjCMethodParameter()); ParmVarDeclBits.addBits(D->getFunctionScopeDepth(), /*BitsWidth=*/7); ParmVarDeclBits.addBits(D->getFunctionScopeIndex(), /*BitsWidth=*/8); + // FIXME: stable encoding + ParmVarDeclBits.addBits(D->getObjCDeclQualifier(), /*BitsWidth=*/7); ParmVarDeclBits.addBit(D->isKNRPromoted()); ParmVarDeclBits.addBit(D->hasInheritedDefaultArg()); ParmVarDeclBits.addBit(D->hasUninstantiatedDefaultArg()); + ParmVarDeclBits.addBit(D->getExplicitObjectParamThisLoc().isValid()); Record.push_back(ParmVarDeclBits); - Record.push_back(D->getObjCDeclQualifier()); // FIXME: stable encoding if (D->hasUninstantiatedDefaultArg()) Record.AddStmt(D->getUninstantiatedDefaultArg()); - Record.AddSourceLocation(D->getExplicitObjectParamThisLoc()); + if (D->getExplicitObjectParamThisLoc().isValid()) + Record.AddSourceLocation(D->getExplicitObjectParamThisLoc()); Code = serialization::DECL_PARM_VAR; // If the assumptions about the DECL_PARM_VAR abbrev are true, use it. Here @@ -2038,13 +2040,12 @@ void ASTWriter::WriteDeclAbbrevs() { Abv = std::make_shared(); Abv->Add(BitCodeAbbrevOp(serialization::DECL_FIELD)); // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2068,13 +2069,12 @@ void ASTWriter::WriteDeclAbbrevs() { Abv = std::make_shared(); Abv->Add(BitCodeAbbrevOp(serialization::DECL_OBJC_IVAR)); // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2103,13 +2103,12 @@ void ASTWriter::WriteDeclAbbrevs() { // Redeclarable Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2122,11 +2121,11 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // IdentifierNamespace Abv->Add(BitCodeAbbrevOp( BitCodeAbbrevOp::Fixed, - 7)); // Packed Tag Decl Bits: getTagKind, isCompleteDefinition, - // EmbeddedInDeclarator, IsFreeStanding, isCompleteDefinitionRequired + 9)); // Packed Tag Decl Bits: getTagKind, isCompleteDefinition, + // EmbeddedInDeclarator, IsFreeStanding, + // isCompleteDefinitionRequired, ExtInfoKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SourceLocation Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SourceLocation - Abv->Add(BitCodeAbbrevOp(0)); // ExtInfoKind // EnumDecl Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // AddTypeRef Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // IntegerType @@ -2145,13 +2144,12 @@ void ASTWriter::WriteDeclAbbrevs() { // Redeclarable Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2164,11 +2162,11 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // IdentifierNamespace Abv->Add(BitCodeAbbrevOp( BitCodeAbbrevOp::Fixed, - 7)); // Packed Tag Decl Bits: getTagKind, isCompleteDefinition, - // EmbeddedInDeclarator, IsFreeStanding, isCompleteDefinitionRequired + 9)); // Packed Tag Decl Bits: getTagKind, isCompleteDefinition, + // EmbeddedInDeclarator, IsFreeStanding, + // isCompleteDefinitionRequired, ExtInfoKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SourceLocation Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SourceLocation - Abv->Add(BitCodeAbbrevOp(0)); // ExtInfoKind // RecordDecl Abv->Add(BitCodeAbbrevOp( BitCodeAbbrevOp::Fixed, @@ -2194,13 +2192,12 @@ void ASTWriter::WriteDeclAbbrevs() { // Redeclarable Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2221,10 +2218,9 @@ void ASTWriter::WriteDeclAbbrevs() { // ParmVarDecl Abv->Add(BitCodeAbbrevOp( BitCodeAbbrevOp::Fixed, - 19)); // Packed Parm Var Decl bits: IsObjCMethodParameter, ScopeDepth, - // ScopeIndex, KNRPromoted, HasInheritedDefaultArg - Abv->Add(BitCodeAbbrevOp(0)); // ObjCDeclQualifier - Abv->Add(BitCodeAbbrevOp(0)); // HasUninstantiatedDefaultArg + 27)); // Packed Parm Var Decl bits: IsObjCMethodParameter, ScopeDepth, + // ScopeIndex, ObjCDeclQualifier, KNRPromoted, + // HasInheritedDefaultArg, HasUninstantiatedDefaultArg // Type Source Info Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array)); Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // TypeLoc @@ -2236,13 +2232,12 @@ void ASTWriter::WriteDeclAbbrevs() { // Redeclarable Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2262,13 +2257,12 @@ void ASTWriter::WriteDeclAbbrevs() { // Redeclarable Abv->Add(BitCodeAbbrevOp(0)); // No redeclaration // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(0)); // NameKind = Identifier @@ -2303,13 +2297,12 @@ void ASTWriter::WriteDeclAbbrevs() { // FIXME: Implement abbreviation for other template kinds. Abv->Add(BitCodeAbbrevOp(FunctionDecl::TK_NonTemplate)); // TemplateKind // Decl + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, + 12)); // Packed DeclBits: HasStandaloneLexicalDC, + // isInvalidDecl, HasAttrs, isImplicit, isUsed, + // isReferenced, TopLevelDeclInObjCContainer, + // AccessSpecifier, ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // DeclContext - Abv->Add(BitCodeAbbrevOp(0)); // LexicalDeclContext - Abv->Add(BitCodeAbbrevOp( - BitCodeAbbrevOp::Fixed, - 11)); // Packed DeclBits: isInvalidDecl, HasAttrs, isImplicit, isUsed, - // isReferenced, TopLevelDeclInObjCContainer, AccessSpecifier, - // ModuleOwnershipKind Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // SubmoduleID // NamedDecl Abv->Add(BitCodeAbbrevOp(DeclarationName::Identifier)); // NameKind @@ -2346,16 +2339,14 @@ void ASTWriter::WriteDeclAbbrevs() { Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); DeclCXXMethodAbbrev = Stream.EmitAbbrev(std::move(Abv)); - unsigned ExprDependenceBits = llvm::BitWidth; // Abbreviation for EXPR_DECL_REF Abv = std::make_shared(); Abv->Add(BitCodeAbbrevOp(serialization::EXPR_DECL_REF)); //Stmt // Expr Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, ExprDependenceBits)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetValueKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetObjectKind + // DependenceKind, ValueKind, ObjectKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); //DeclRefExpr Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //HasQualifier Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); //GetDeclFound @@ -2374,9 +2365,8 @@ void ASTWriter::WriteDeclAbbrevs() { //Stmt // Expr Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, ExprDependenceBits)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetValueKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetObjectKind + // DependenceKind, ValueKind, ObjectKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); //Integer Literal Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location Abv->Add(BitCodeAbbrevOp(32)); // Bit Width @@ -2389,9 +2379,8 @@ void ASTWriter::WriteDeclAbbrevs() { //Stmt // Expr Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, ExprDependenceBits)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetValueKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetObjectKind + // DependenceKind, ValueKind, ObjectKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); //Character Literal Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // getValue Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Location @@ -2404,9 +2393,8 @@ void ASTWriter::WriteDeclAbbrevs() { // Stmt // Expr Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // Type - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, ExprDependenceBits)); - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetValueKind - Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 3)); //GetObjectKind + // DependenceKind, ValueKind, ObjectKind + Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 10)); // CastExpr Abv->Add(BitCodeAbbrevOp(0)); // PathSize Abv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Fixed, 1)); // HasFPFeatures diff --git a/clang/lib/Serialization/ASTWriterStmt.cpp b/clang/lib/Serialization/ASTWriterStmt.cpp index 59be6828fafa..8524484ea8a0 100644 --- a/clang/lib/Serialization/ASTWriterStmt.cpp +++ b/clang/lib/Serialization/ASTWriterStmt.cpp @@ -549,9 +549,14 @@ void ASTStmtWriter::VisitCapturedStmt(CapturedStmt *S) { void ASTStmtWriter::VisitExpr(Expr *E) { VisitStmt(E); Record.AddTypeRef(E->getType()); - Record.push_back(E->getDependence()); - Record.push_back(E->getValueKind()); - Record.push_back(E->getObjectKind()); + + BitsPacker ExprBits; + + ExprBits.addBits(E->getDependence(), /*BitsWidth=*/5); + ExprBits.addBits(E->getValueKind(), /*BitsWidth=*/2); + ExprBits.addBits(E->getObjectKind(), /*BitsWidth=*/3); + + Record.push_back(ExprBits); } void ASTStmtWriter::VisitConstantExpr(ConstantExpr *E) { @@ -866,14 +871,20 @@ void ASTStmtWriter::VisitOMPIteratorExpr(OMPIteratorExpr *E) { void ASTStmtWriter::VisitCallExpr(CallExpr *E) { VisitExpr(E); - Record.push_back(E->getNumArgs()); - Record.push_back(E->hasStoredFPFeatures()); + + BitsPacker CallExprBits; + // 16 bits should be sufficient to store the number args; + CallExprBits.addBits(E->getNumArgs(), /*BitsWidth=*/16); + CallExprBits.addBit(E->hasStoredFPFeatures()); + CallExprBits.addBit(static_cast(E->getADLCallKind())); + Record.push_back(CallExprBits); + Record.AddSourceLocation(E->getRParenLoc()); Record.AddStmt(E->getCallee()); for (CallExpr::arg_iterator Arg = E->arg_begin(), ArgEnd = E->arg_end(); Arg != ArgEnd; ++Arg) Record.AddStmt(*Arg); - Record.push_back(static_cast(E->getADLCallKind())); + if (E->hasStoredFPFeatures()) Record.push_back(E->getFPFeatures().getAsOpaqueInt()); Code = serialization::EXPR_CALL; @@ -1938,14 +1949,19 @@ ASTStmtWriter::VisitCXXUnresolvedConstructExpr(CXXUnresolvedConstructExpr *E) { void ASTStmtWriter::VisitOverloadExpr(OverloadExpr *E) { VisitExpr(E); - Record.push_back(E->getNumDecls()); - Record.push_back(E->hasTemplateKWAndArgsInfo()); + BitsPacker OverloadExprBits; + // 14 Bits should enough to store the number of decls. + OverloadExprBits.addBits(E->getNumDecls(), /*BitWidth=*/14); + OverloadExprBits.addBit(E->hasTemplateKWAndArgsInfo()); if (E->hasTemplateKWAndArgsInfo()) { const ASTTemplateKWAndArgsInfo &ArgInfo = *E->getTrailingASTTemplateKWAndArgsInfo(); - Record.push_back(ArgInfo.NumTemplateArgs); + // 14 Bits should enough to store the number of template args. + OverloadExprBits.addBits(ArgInfo.NumTemplateArgs, /*BitWidth=*/14); + Record.push_back(OverloadExprBits); AddTemplateKWAndArgsInfo(ArgInfo, E->getTrailingTemplateArgumentLoc()); - } + } else + Record.push_back(OverloadExprBits); for (OverloadExpr::decls_iterator OvI = E->decls_begin(), OvE = E->decls_end(); diff --git a/clang/lib/Serialization/GlobalModuleIndex.cpp b/clang/lib/Serialization/GlobalModuleIndex.cpp index fb80a1998d0e..dd4fc3e00905 100644 --- a/clang/lib/Serialization/GlobalModuleIndex.cpp +++ b/clang/lib/Serialization/GlobalModuleIndex.cpp @@ -342,8 +342,8 @@ bool GlobalModuleIndex::loadedModuleFile(ModuleFile *File) { // If the size and modification time match what we expected, record this // module file. bool Failed = true; - if (File->File->getSize() == Info.Size && - File->File->getModificationTime() == Info.ModTime) { + if (File->File.getSize() == Info.Size && + File->File.getModificationTime() == Info.ModTime) { Info.File = File; ModulesByFile[File] = Known->second; diff --git a/clang/lib/Serialization/ModuleManager.cpp b/clang/lib/Serialization/ModuleManager.cpp index de4cd3d05853..51b642941296 100644 --- a/clang/lib/Serialization/ModuleManager.cpp +++ b/clang/lib/Serialization/ModuleManager.cpp @@ -52,8 +52,8 @@ ModuleFile *ModuleManager::lookupByFileName(StringRef Name) const { ModuleFile *ModuleManager::lookupByModuleName(StringRef Name) const { if (const Module *Mod = HeaderSearchInfo.getModuleMap().findModule(Name)) - if (const FileEntry *File = Mod->getASTFile()) - return lookup(File); + if (OptionalFileEntryRef File = Mod->getASTFile()) + return lookup(*File); return nullptr; } @@ -108,7 +108,7 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, // Look for the file entry. This only fails if the expected size or // modification time differ. - OptionalFileEntryRefDegradesToFileEntryPtr Entry; + OptionalFileEntryRef Entry; if (Type == MK_ExplicitModule || Type == MK_PrebuiltModule) { // If we're not expecting to pull this file out of the module cache, it // might have a different mtime due to being moved across filesystems in @@ -123,7 +123,7 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, return OutOfDate; } - if (!Entry && FileName != "-") { + if (!Entry) { ErrorStr = "module file not found"; return Missing; } @@ -150,7 +150,7 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, }; // Check whether we already loaded this module, before - if (ModuleFile *ModuleEntry = Modules.lookup(Entry)) { + if (ModuleFile *ModuleEntry = Modules.lookup(*Entry)) { if (implicitModuleNamesMatch(Type, ModuleEntry, *Entry)) { // Check the stored signature. if (checkSignature(ModuleEntry->Signature, ExpectedSignature, ErrorStr)) @@ -163,10 +163,9 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, } // Allocate a new module. - auto NewModule = std::make_unique(Type, Generation); + auto NewModule = std::make_unique(Type, *Entry, Generation); NewModule->Index = Chain.size(); NewModule->FileName = FileName.str(); - NewModule->File = Entry; NewModule->ImportLoc = ImportLoc; NewModule->InputFilesValidationTimestamp = 0; @@ -198,21 +197,15 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, Entry->closeFile(); return OutOfDate; } else { - // Open the AST file. - llvm::ErrorOr> Buf((std::error_code())); - if (FileName == "-") { - Buf = llvm::MemoryBuffer::getSTDIN(); - } else { - // Get a buffer of the file and close the file descriptor when done. - // The file is volatile because in a parallel build we expect multiple - // compiler processes to use the same module file rebuilding it if needed. - // - // RequiresNullTerminator is false because module files don't need it, and - // this allows the file to still be mmapped. - Buf = FileMgr.getBufferForFile(*NewModule->File, - /*IsVolatile=*/true, - /*RequiresNullTerminator=*/false); - } + // Get a buffer of the file and close the file descriptor when done. + // The file is volatile because in a parallel build we expect multiple + // compiler processes to use the same module file rebuilding it if needed. + // + // RequiresNullTerminator is false because module files don't need it, and + // this allows the file to still be mmapped. + auto Buf = FileMgr.getBufferForFile(NewModule->File, + /*IsVolatile=*/true, + /*RequiresNullTerminator=*/false); if (!Buf) { ErrorStr = Buf.getError().message(); @@ -232,7 +225,7 @@ ModuleManager::addModule(StringRef FileName, ModuleKind Type, return OutOfDate; // We're keeping this module. Store it everywhere. - Module = Modules[Entry] = NewModule.get(); + Module = Modules[*Entry] = NewModule.get(); updateModuleImports(*NewModule, ImportedBy, ImportLoc); @@ -441,22 +434,19 @@ void ModuleManager::visit(llvm::function_ref Visitor, bool ModuleManager::lookupModuleFile(StringRef FileName, off_t ExpectedSize, time_t ExpectedModTime, OptionalFileEntryRef &File) { - File = std::nullopt; - if (FileName == "-") + if (FileName == "-") { + File = expectedToOptional(FileMgr.getSTDIN()); return false; + } // Open the file immediately to ensure there is no race between stat'ing and // opening the file. - OptionalFileEntryRef FileOrErr = - expectedToOptional(FileMgr.getFileRef(FileName, /*OpenFile=*/true, - /*CacheFailure=*/false)); - if (!FileOrErr) - return false; - - File = *FileOrErr; + File = FileMgr.getOptionalFileRef(FileName, /*OpenFile=*/true, + /*CacheFailure=*/false); - if ((ExpectedSize && ExpectedSize != File->getSize()) || - (ExpectedModTime && ExpectedModTime != File->getModificationTime())) + if (File && + ((ExpectedSize && ExpectedSize != File->getSize()) || + (ExpectedModTime && ExpectedModTime != File->getModificationTime()))) // Do not destroy File, as it may be referenced. If we need to rebuild it, // it will be destroyed by removeModules. return true; diff --git a/clang/lib/StaticAnalyzer/Checkers/BitwiseShiftChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/BitwiseShiftChecker.cpp index d4aa9fa1339f..339927c165fe 100644 --- a/clang/lib/StaticAnalyzer/Checkers/BitwiseShiftChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/BitwiseShiftChecker.cpp @@ -344,7 +344,7 @@ BitwiseShiftValidator::createBugReport(StringRef ShortMsg, StringRef Msg) const } // anonymous namespace class BitwiseShiftChecker : public Checker> { - mutable std::unique_ptr BTPtr; + BugType BT{this, "Bitwise shift", "Suspicious operation"}; public: void checkPreStmt(const BinaryOperator *B, CheckerContext &Ctx) const { @@ -353,11 +353,7 @@ public: if (Op != BO_Shl && Op != BO_Shr) return; - if (!BTPtr) - BTPtr = std::make_unique(this, "Bitwise shift", - "Suspicious operation"); - - BitwiseShiftValidator(B, Ctx, *BTPtr, Pedantic).run(); + BitwiseShiftValidator(B, Ctx, BT, Pedantic).run(); } bool Pedantic = false; diff --git a/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp b/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp index 5844f4399100..14433d06c2d0 100644 --- a/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp +++ b/clang/lib/StaticAnalyzer/Checkers/EnumCastOutOfRangeChecker.cpp @@ -22,10 +22,12 @@ #include "clang/StaticAnalyzer/Checkers/BuiltinCheckerRegistration.h" #include "clang/StaticAnalyzer/Core/BugReporter/BugType.h" #include "clang/StaticAnalyzer/Core/PathSensitive/CheckerContext.h" +#include "llvm/Support/FormatVariadic.h" #include using namespace clang; using namespace ento; +using llvm::formatv; namespace { // This evaluator checks two SVals for equality. The first SVal is provided via @@ -87,17 +89,22 @@ void EnumCastOutOfRangeChecker::reportWarning(CheckerContext &C, EnumValueCastOutOfRange.reset( new BugType(this, "Enum cast out of range")); - llvm::SmallString<128> Msg{"The value provided to the cast expression is " - "not in the valid range of values for "}; - StringRef EnumName{E->getName()}; - if (EnumName.empty()) { - Msg += "the enum"; - } else { - Msg += '\''; - Msg += EnumName; - Msg += '\''; + std::string ValueStr = "", NameStr = "the enum"; + + // Try to add details to the message: + const auto ConcreteValue = + C.getSVal(CE->getSubExpr()).getAs(); + if (ConcreteValue) { + ValueStr = formatv(" '{0}'", ConcreteValue->getValue()); + } + if (StringRef EnumName{E->getName()}; !EnumName.empty()) { + NameStr = formatv("'{0}'", EnumName); } + std::string Msg = formatv("The value{0} provided to the cast expression is " + "not in the valid range of values for {1}", + ValueStr, NameStr); + auto BR = std::make_unique(*EnumValueCastOutOfRange, Msg, N); bugreporter::trackExpressionValue(N, CE->getSubExpr(), *BR); diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp index 1058ddb8254c..f65da413bb87 100644 --- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp +++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp @@ -521,7 +521,7 @@ ModuleDepCollectorPP::handleTopLevelModule(const Module *M) { serialization::ModuleFile *MF = MDC.ScanInstance.getASTReader()->getModuleManager().lookup( - M->getASTFile()); + *M->getASTFile()); MDC.ScanInstance.getASTReader()->visitInputFileInfos( *MF, /*IncludeSystem=*/true, [&](const serialization::InputFileInfo &IFI, bool IsSystem) { diff --git a/clang/test/AST/HLSL/RWBuffer-AST.hlsl b/clang/test/AST/HLSL/RWBuffer-AST.hlsl index 9dd9244b73ee..c1613520a146 100644 --- a/clang/test/AST/HLSL/RWBuffer-AST.hlsl +++ b/clang/test/AST/HLSL/RWBuffer-AST.hlsl @@ -13,8 +13,6 @@ // EMPTY: ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <> implicit RWBuffer // EMPTY-NEXT: TemplateTypeParmDecl 0x{{[0-9A-Fa-f]+}} <> class depth 0 index 0 element_type -// EMPTY-NEXT: TemplateArgument type 'float' -// EMPTY-NEXT: BuiltinType 0x{{[0-9A-Fa-f]+}} 'float' // EMPTY-NEXT: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <> implicit class RWBuffer // EMPTY-NEXT: FinalAttr 0x{{[0-9A-Fa-f]+}} <> Implicit final @@ -33,8 +31,6 @@ RWBuffer Buffer; // CHECK: ClassTemplateDecl 0x{{[0-9A-Fa-f]+}} <> implicit RWBuffer // CHECK-NEXT: TemplateTypeParmDecl 0x{{[0-9A-Fa-f]+}} <> class depth 0 index 0 element_type -// CHECK-NEXT: TemplateArgument type 'float' -// CHECK-NEXT: BuiltinType 0x{{[0-9A-Fa-f]+}} 'float' // CHECK-NEXT: CXXRecordDecl 0x{{[0-9A-Fa-f]+}} <> implicit class RWBuffer definition // CHECK: FinalAttr 0x{{[0-9A-Fa-f]+}} <> Implicit final diff --git a/clang/test/AST/HLSL/pch.hlsl b/clang/test/AST/HLSL/pch.hlsl index 74254fedf27c..27fae8f499da 100644 --- a/clang/test/AST/HLSL/pch.hlsl +++ b/clang/test/AST/HLSL/pch.hlsl @@ -6,7 +6,7 @@ // Make sure PCH works by using function declared in PCH header and declare a RWBuffer in current file. // CHECK:FunctionDecl 0x[[FOO:[0-9a-f]+]] <{{.*}}:2:1, line:4:1> line:2:8 imported used foo 'float2 (float2, float2)' -// CHECK:VarDecl 0x{{[0-9a-f]+}} <{{.*}}:10:1, col:23> col:23 Buffer 'hlsl::RWBuffer':'hlsl::RWBuffer<>' +// CHECK:VarDecl 0x{{[0-9a-f]+}} <{{.*}}:10:1, col:23> col:23 Buffer 'hlsl::RWBuffer' hlsl::RWBuffer Buffer; float2 bar(float2 a, float2 b) { diff --git a/clang/test/AST/HLSL/pch_with_buf.hlsl b/clang/test/AST/HLSL/pch_with_buf.hlsl index 4e657606cbcb..e8eae533af62 100644 --- a/clang/test/AST/HLSL/pch_with_buf.hlsl +++ b/clang/test/AST/HLSL/pch_with_buf.hlsl @@ -5,9 +5,9 @@ // Make sure PCH works by using function declared in PCH header. // CHECK:FunctionDecl 0x[[FOO:[0-9a-f]+]] <{{.*}}:2:1, line:4:1> line:2:8 imported used foo 'float2 (float2, float2)' // Make sure buffer defined in PCH works. -// CHECK:VarDecl 0x{{[0-9a-f]+}} col:17 imported Buf 'RWBuffer':'hlsl::RWBuffer<>' +// CHECK:VarDecl 0x{{[0-9a-f]+}} col:17 imported Buf 'RWBuffer' // Make sure declare a RWBuffer in current file works. -// CHECK:VarDecl 0x{{[0-9a-f]+}} <{{.*}}:11:1, col:23> col:23 Buf2 'hlsl::RWBuffer':'hlsl::RWBuffer<>' +// CHECK:VarDecl 0x{{[0-9a-f]+}} <{{.*}}:11:1, col:23> col:23 Buf2 'hlsl::RWBuffer' hlsl::RWBuffer Buf2; float2 bar(float2 a, float2 b) { diff --git a/clang/test/AST/HLSL/resource_binding_attr.hlsl b/clang/test/AST/HLSL/resource_binding_attr.hlsl index 683607197111..71900f2dbda5 100644 --- a/clang/test/AST/HLSL/resource_binding_attr.hlsl +++ b/clang/test/AST/HLSL/resource_binding_attr.hlsl @@ -23,15 +23,15 @@ float foo() { return a + b; } -// CHECK: VarDecl 0x{{[0-9a-f]+}} <{{.*}}> col:17 UAV 'RWBuffer':'hlsl::RWBuffer<>' callinit -// CHECK-NEXT:-CXXConstructExpr 0x{{[0-9a-f]+}} 'RWBuffer':'hlsl::RWBuffer<>' 'void ()' +// CHECK: VarDecl 0x{{[0-9a-f]+}} <{{.*}}> col:17 UAV 'RWBuffer':'hlsl::RWBuffer' callinit +// CHECK-NEXT:-CXXConstructExpr 0x{{[0-9a-f]+}} 'RWBuffer':'hlsl::RWBuffer' 'void ()' // CHECK-NEXT:-HLSLResourceBindingAttr 0x{{[0-9a-f]+}} "u3" "space0" RWBuffer UAV : register(u3); -// CHECK: -VarDecl 0x{{[0-9a-f]+}} <{{.*}}> col:17 UAV1 'RWBuffer':'hlsl::RWBuffer<>' callinit -// CHECK-NEXT:-CXXConstructExpr 0x{{[0-9a-f]+}} 'RWBuffer':'hlsl::RWBuffer<>' 'void ()' +// CHECK: -VarDecl 0x{{[0-9a-f]+}} <{{.*}}> col:17 UAV1 'RWBuffer':'hlsl::RWBuffer' callinit +// CHECK-NEXT:-CXXConstructExpr 0x{{[0-9a-f]+}} 'RWBuffer':'hlsl::RWBuffer' 'void ()' // CHECK-NEXT:-HLSLResourceBindingAttr 0x{{[0-9a-f]+}} "u2" "space0" -// CHECK-NEXT:-VarDecl 0x{{[0-9a-f]+}} col:38 UAV2 'RWBuffer':'hlsl::RWBuffer<>' callinit -// CHECK-NEXT:-CXXConstructExpr 0x{{[0-9a-f]+}} 'RWBuffer':'hlsl::RWBuffer<>' 'void ()' +// CHECK-NEXT:-VarDecl 0x{{[0-9a-f]+}} col:38 UAV2 'RWBuffer':'hlsl::RWBuffer' callinit +// CHECK-NEXT:-CXXConstructExpr 0x{{[0-9a-f]+}} 'RWBuffer':'hlsl::RWBuffer' 'void ()' // CHECK-NEXT:-HLSLResourceBindingAttr 0x{{[0-9a-f]+}} "u4" "space0" RWBuffer UAV1 : register(u2), UAV2 : register(u4); diff --git a/clang/test/AST/Interp/builtin-functions.cpp b/clang/test/AST/Interp/builtin-functions.cpp index 0726dab37cb4..35a1f9a75092 100644 --- a/clang/test/AST/Interp/builtin-functions.cpp +++ b/clang/test/AST/Interp/builtin-functions.cpp @@ -331,3 +331,11 @@ namespace bitreverse { char bitreverse3[__builtin_bitreverse32(0x12345678) == 0x1E6A2C48 ? 1 : -1]; char bitreverse4[__builtin_bitreverse64(0x0123456789ABCDEFULL) == 0xF7B3D591E6A2C480 ? 1 : -1]; } + +namespace expect { + constexpr int a() { + return 12; + } + static_assert(__builtin_expect(a(),1) == 12, ""); + static_assert(__builtin_expect_with_probability(a(), 1, 1.0) == 12, ""); +} diff --git a/clang/test/AST/Interp/floats.cpp b/clang/test/AST/Interp/floats.cpp index e17167f5bf6d..45c31c759e47 100644 --- a/clang/test/AST/Interp/floats.cpp +++ b/clang/test/AST/Interp/floats.cpp @@ -39,6 +39,10 @@ constexpr float m = 5.0f / 0.0f; // ref-error {{must be initialized by a constan static_assert(~2.0f == 3, ""); // ref-error {{invalid argument type 'float' to unary expression}} \ // expected-error {{invalid argument type 'float' to unary expression}} + +typedef int tdb[(long long)4e20]; //expected-error {{variable length}} \ + //ref-error {{variable length}} + /// Initialized by a double. constexpr float df = 0.0; /// The other way around. diff --git a/clang/test/AST/ms-constexpr.cpp b/clang/test/AST/ms-constexpr.cpp new file mode 100644 index 000000000000..e85af8494f33 --- /dev/null +++ b/clang/test/AST/ms-constexpr.cpp @@ -0,0 +1,28 @@ +// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.33 -std=c++20 -ast-dump -verify %s | FileCheck %s +// expected-no-diagnostics + +// CHECK: used f1 'bool ()' +// CHECK: MSConstexprAttr 0x{{[0-9a-f]+}} +[[msvc::constexpr]] bool f1() { return true; } + +// CHECK: used constexpr f2 'bool ()' +// CHECK-NEXT: CompoundStmt 0x{{[0-9a-f]+}} +// CHECK-NEXT: AttributedStmt 0x{{[0-9a-f]+}} +// CHECK-NEXT: MSConstexprAttr 0x{{[0-9a-f]+}} +// CHECK-NEXT: ReturnStmt 0x{{[0-9a-f]+}} +constexpr bool f2() { [[msvc::constexpr]] return f1(); } +static_assert(f2()); + +struct S1 { + // CHECK: used vm 'bool ()' virtual + // CHECK: MSConstexprAttr 0x{{[0-9a-f]+}} + [[msvc::constexpr]] virtual bool vm() { return true; } + + // CHECK: used constexpr cm 'bool ()' + // CHECK-NEXT: CompoundStmt 0x{{[0-9a-f]+}} + // CHECK-NEXT: AttributedStmt 0x{{[0-9a-f]+}} + // CHECK-NEXT: MSConstexprAttr 0x{{[0-9a-f]+}} + // CHECK-NEXT: ReturnStmt 0x{{[0-9a-f]+}} + constexpr bool cm() { [[msvc::constexpr]] return vm(); } +}; +static_assert(S1{}.cm()); diff --git a/clang/test/Analysis/enum-cast-out-of-range.cpp b/clang/test/Analysis/enum-cast-out-of-range.cpp index 0eb740664ecd..09835d420672 100644 --- a/clang/test/Analysis/enum-cast-out-of-range.cpp +++ b/clang/test/Analysis/enum-cast-out-of-range.cpp @@ -43,115 +43,115 @@ struct S { }; void unscopedUnspecified() { - unscoped_unspecified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning {{The value '-5' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} unscoped_unspecified_t ValidNegativeValue1 = static_cast(-4); // OK. unscoped_unspecified_t ValidNegativeValue2 = static_cast(-3); // OK. - unscoped_unspecified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} - unscoped_unspecified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} - unscoped_unspecified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} unscoped_unspecified_t ValidPositiveValue1 = static_cast(1); // OK. unscoped_unspecified_t ValidPositiveValue2 = static_cast(2); // OK. - unscoped_unspecified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} unscoped_unspecified_t ValidPositiveValue3 = static_cast(4); // OK. - unscoped_unspecified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} } void unscopedSpecified() { - unscoped_specified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning {{The value '-5' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} unscoped_specified_t ValidNegativeValue1 = static_cast(-4); // OK. unscoped_specified_t ValidNegativeValue2 = static_cast(-3); // OK. - unscoped_specified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} - unscoped_specified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} - unscoped_specified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} unscoped_specified_t ValidPositiveValue1 = static_cast(1); // OK. unscoped_specified_t ValidPositiveValue2 = static_cast(2); // OK. - unscoped_specified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} unscoped_specified_t ValidPositiveValue3 = static_cast(4); // OK. - unscoped_specified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} } void scopedUnspecified() { - scoped_unspecified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning{{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning{{The value '-5' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} scoped_unspecified_t ValidNegativeValue1 = static_cast(-4); // OK. scoped_unspecified_t ValidNegativeValue2 = static_cast(-3); // OK. - scoped_unspecified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} - scoped_unspecified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} - scoped_unspecified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} scoped_unspecified_t ValidPositiveValue1 = static_cast(1); // OK. scoped_unspecified_t ValidPositiveValue2 = static_cast(2); // OK. - scoped_unspecified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} scoped_unspecified_t ValidPositiveValue3 = static_cast(4); // OK. - scoped_unspecified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} } void scopedSpecified() { - scoped_specified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidBeforeRangeBegin = static_cast(-5); // expected-warning {{The value '-5' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} scoped_specified_t ValidNegativeValue1 = static_cast(-4); // OK. scoped_specified_t ValidNegativeValue2 = static_cast(-3); // OK. - scoped_specified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} - scoped_specified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} - scoped_specified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange1 = static_cast(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange2 = static_cast(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange3 = static_cast(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} scoped_specified_t ValidPositiveValue1 = static_cast(1); // OK. scoped_specified_t ValidPositiveValue2 = static_cast(2); // OK. - scoped_specified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange4 = static_cast(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} scoped_specified_t ValidPositiveValue3 = static_cast(4); // OK. - scoped_specified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidAfterRangeEnd = static_cast(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} } void unscopedUnspecifiedCStyle() { - unscoped_unspecified_t InvalidBeforeRangeBegin = (unscoped_unspecified_t)(-5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidBeforeRangeBegin = (unscoped_unspecified_t)(-5); // expected-warning {{The value '-5' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} unscoped_unspecified_t ValidNegativeValue1 = (unscoped_unspecified_t)(-4); // OK. unscoped_unspecified_t ValidNegativeValue2 = (unscoped_unspecified_t)(-3); // OK. - unscoped_unspecified_t InvalidInsideRange1 = (unscoped_unspecified_t)(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} - unscoped_unspecified_t InvalidInsideRange2 = (unscoped_unspecified_t)(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} - unscoped_unspecified_t InvalidInsideRange3 = (unscoped_unspecified_t)(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange1 = (unscoped_unspecified_t)(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange2 = (unscoped_unspecified_t)(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange3 = (unscoped_unspecified_t)(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} unscoped_unspecified_t ValidPositiveValue1 = (unscoped_unspecified_t)(1); // OK. unscoped_unspecified_t ValidPositiveValue2 = (unscoped_unspecified_t)(2); // OK. - unscoped_unspecified_t InvalidInsideRange4 = (unscoped_unspecified_t)(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidInsideRange4 = (unscoped_unspecified_t)(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} unscoped_unspecified_t ValidPositiveValue3 = (unscoped_unspecified_t)(4); // OK. - unscoped_unspecified_t InvalidAfterRangeEnd = (unscoped_unspecified_t)(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + unscoped_unspecified_t InvalidAfterRangeEnd = (unscoped_unspecified_t)(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} } void unscopedSpecifiedCStyle() { - unscoped_specified_t InvalidBeforeRangeBegin = (unscoped_specified_t)(-5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidBeforeRangeBegin = (unscoped_specified_t)(-5); // expected-warning {{The value '-5' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} unscoped_specified_t ValidNegativeValue1 = (unscoped_specified_t)(-4); // OK. unscoped_specified_t ValidNegativeValue2 = (unscoped_specified_t)(-3); // OK. - unscoped_specified_t InvalidInsideRange1 = (unscoped_specified_t)(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} - unscoped_specified_t InvalidInsideRange2 = (unscoped_specified_t)(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} - unscoped_specified_t InvalidInsideRange3 = (unscoped_specified_t)(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange1 = (unscoped_specified_t)(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange2 = (unscoped_specified_t)(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange3 = (unscoped_specified_t)(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} unscoped_specified_t ValidPositiveValue1 = (unscoped_specified_t)(1); // OK. unscoped_specified_t ValidPositiveValue2 = (unscoped_specified_t)(2); // OK. - unscoped_specified_t InvalidInsideRange4 = (unscoped_specified_t)(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidInsideRange4 = (unscoped_specified_t)(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} unscoped_specified_t ValidPositiveValue3 = (unscoped_specified_t)(4); // OK. - unscoped_specified_t InvalidAfterRangeEnd = (unscoped_specified_t)(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} + unscoped_specified_t InvalidAfterRangeEnd = (unscoped_specified_t)(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'unscoped_specified_t'}} } void scopedUnspecifiedCStyle() { - scoped_unspecified_t InvalidBeforeRangeBegin = (scoped_unspecified_t)(-5); // expected-warning{{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidBeforeRangeBegin = (scoped_unspecified_t)(-5); // expected-warning{{The value '-5' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} scoped_unspecified_t ValidNegativeValue1 = (scoped_unspecified_t)(-4); // OK. scoped_unspecified_t ValidNegativeValue2 = (scoped_unspecified_t)(-3); // OK. - scoped_unspecified_t InvalidInsideRange1 = (scoped_unspecified_t)(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} - scoped_unspecified_t InvalidInsideRange2 = (scoped_unspecified_t)(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} - scoped_unspecified_t InvalidInsideRange3 = (scoped_unspecified_t)(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange1 = (scoped_unspecified_t)(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange2 = (scoped_unspecified_t)(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange3 = (scoped_unspecified_t)(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} scoped_unspecified_t ValidPositiveValue1 = (scoped_unspecified_t)(1); // OK. scoped_unspecified_t ValidPositiveValue2 = (scoped_unspecified_t)(2); // OK. - scoped_unspecified_t InvalidInsideRange4 = (scoped_unspecified_t)(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidInsideRange4 = (scoped_unspecified_t)(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} scoped_unspecified_t ValidPositiveValue3 = (scoped_unspecified_t)(4); // OK. - scoped_unspecified_t InvalidAfterRangeEnd = (scoped_unspecified_t)(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} + scoped_unspecified_t InvalidAfterRangeEnd = (scoped_unspecified_t)(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'scoped_unspecified_t'}} } void scopedSpecifiedCStyle() { - scoped_specified_t InvalidBeforeRangeBegin = (scoped_specified_t)(-5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidBeforeRangeBegin = (scoped_specified_t)(-5); // expected-warning {{The value '-5' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} scoped_specified_t ValidNegativeValue1 = (scoped_specified_t)(-4); // OK. scoped_specified_t ValidNegativeValue2 = (scoped_specified_t)(-3); // OK. - scoped_specified_t InvalidInsideRange1 = (scoped_specified_t)(-2); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} - scoped_specified_t InvalidInsideRange2 = (scoped_specified_t)(-1); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} - scoped_specified_t InvalidInsideRange3 = (scoped_specified_t)(0); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange1 = (scoped_specified_t)(-2); // expected-warning {{The value '-2' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange2 = (scoped_specified_t)(-1); // expected-warning {{The value '-1' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange3 = (scoped_specified_t)(0); // expected-warning {{The value '0' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} scoped_specified_t ValidPositiveValue1 = (scoped_specified_t)(1); // OK. scoped_specified_t ValidPositiveValue2 = (scoped_specified_t)(2); // OK. - scoped_specified_t InvalidInsideRange4 = (scoped_specified_t)(3); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidInsideRange4 = (scoped_specified_t)(3); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} scoped_specified_t ValidPositiveValue3 = (scoped_specified_t)(4); // OK. - scoped_specified_t InvalidAfterRangeEnd = (scoped_specified_t)(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + scoped_specified_t InvalidAfterRangeEnd = (scoped_specified_t)(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} } unscoped_unspecified_t unused; @@ -194,13 +194,13 @@ void rangeConstrained6(int input) { void rangeConstrained7(int input) { if (input >= 3 && input <= 3) - auto value = static_cast(input); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} + auto value = static_cast(input); // expected-warning {{The value '3' provided to the cast expression is not in the valid range of values for 'scoped_specified_t'}} } void enumBitFieldAssignment() { S s; s.E = static_cast(4); // OK. - s.E = static_cast(5); // expected-warning {{The value provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} + s.E = static_cast(5); // expected-warning {{The value '5' provided to the cast expression is not in the valid range of values for 'unscoped_unspecified_t'}} } diff --git a/clang/test/C/C2x/n2836_n2939.c b/clang/test/C/C2x/n2836_n2939.c new file mode 100644 index 000000000000..13e23c3def80 --- /dev/null +++ b/clang/test/C/C2x/n2836_n2939.c @@ -0,0 +1,83 @@ +// RUN: %clang_cc1 -x c -std=c2x -fsyntax-only -verify %s +// RUN: %clang_cc1 -x c -std=c2x -E -DPP_ONLY=1 %s | FileCheck %s --strict-whitespace + +/* WG14 N2836: Clang 15 + * Identifier Syntax using Unicode Standard Annex 31 + */ + +/* WG14 N2939: Clang 15 + * Identifier Syntax Fixes + */ + +// Some of the tests below are derived from clang/test/Lexer/unicode.c. + +// This file contains Unicode characters; please do not "fix" them! + +// No diagnostics for pragma directives. +#pragma mark ¡Unicode! + +// lone non-identifier characters are allowed in preprocessing. +#define COPYRIGHT Copyright © 2012 +#define XSTR(X) #X +#define STR(X) XSTR(X) + +static const char *copyright = STR(COPYRIGHT); // no-warning +// CHECK: static const char *copyright = "Copyright © {{2012}}"; + +#if PP_ONLY +COPYRIGHT +// CHECK: Copyright © {{2012}} +#endif + +// The characters in the following identifiers are no longer valid as either +// start or continuation characters as of C23. These are taken from section 1 +// of N2836. +extern int \N{CONSTRUCTION WORKER}; // expected-error {{expected identifier or '('}} +extern int X\N{CONSTRUCTION WORKER}; // expected-error {{character not allowed in an identifier}} +extern int \U0001F477; // expected-error {{expected identifier or '('}} +extern int X\U0001F477; // expected-error {{character not allowed in an identifier}} +extern int 👷; // expected-error {{unexpected character }} \ + // expected-warning {{declaration does not declare anything}} +extern int X👷; // expected-error {{character not allowed in an identifier}} +extern int 🕐; // expected-error {{unexpected character }} \ + // expected-warning {{declaration does not declare anything}} +extern int X🕐; // expected-error {{character not allowed in an identifier}} +extern int 💀; // expected-error {{unexpected character }} \ + // expected-warning {{declaration does not declare anything}} +extern int X💀; // expected-error {{character not allowed in an identifier}} +extern int 👊; // expected-error {{unexpected character }} \ + // expected-warning {{declaration does not declare anything}} +extern int X👊; // expected-error {{character not allowed in an identifier}} +extern int 🚀; // expected-error {{unexpected character }} \ + // expected-warning {{declaration does not declare anything}} +extern int X🚀; // expected-error {{character not allowed in an identifier}} +extern int 😀; // expected-error {{unexpected character }} \ + // expected-warning {{declaration does not declare anything}} +extern int X😀; // expected-error {{character not allowed in an identifier}} + +// The characters in the following identifiers are not allowed as start +// characters, but are allowed as continuation characters. +extern int \N{ARABIC-INDIC DIGIT ZERO}; // expected-error {{expected identifier or '('}} +extern int X\N{ARABIC-INDIC DIGIT ZERO}; +extern int \u0661; // expected-error {{expected identifier or '('}} +extern int X\u0661; +extern int ٢; // expected-error {{character not allowed at the start of an identifier}} \\ + // expected-warning {{declaration does not declare anything}} +extern int X٠; + +// The characters in the following identifiers are not valid start or +// continuation characters in the standard, but are accepted as a conforming +// extension. +extern int \N{SUPERSCRIPT ZERO}; // expected-error {{expected identifier or '('}} +extern int X\N{SUPERSCRIPT ZERO}; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int \u00B9; // expected-error {{expected identifier or '('}} +extern int X\u00B9; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int ²; // expected-error {{character not allowed at the start of an identifier}} \\ + // expected-warning {{declaration does not declare anything}} +extern int X²; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int \N{PARTIAL DIFFERENTIAL}; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int X\N{PARTIAL DIFFERENTIAL}; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int \u2207; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int X\u2207; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int ∞; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} +extern int X∞; // expected-warning {{mathematical notation character in an identifier is a Clang extension}} diff --git a/clang/test/CXX/class.access/class.friend/p6.cpp b/clang/test/CXX/class.access/class.friend/p6.cpp index 2fe20fe77fc8..47104e29dc6b 100644 --- a/clang/test/CXX/class.access/class.friend/p6.cpp +++ b/clang/test/CXX/class.access/class.friend/p6.cpp @@ -22,3 +22,16 @@ void local() { friend void f() { } // expected-error{{friend function cannot be defined in a local class}} }; } + +template void f3(T); + +namespace N { + template void f4(T); +} + +template struct A { + friend void f3(T) {} + friend void f3(T) {} // expected-error{{friend function specialization cannot be defined}} + friend void N::f4(T) {} // expected-error{{friend function definition cannot be qualified with 'N::'}} + friend void N::f4(T) {} // expected-error{{friend function definition cannot be qualified with 'N::'}} +}; diff --git a/clang/test/CXX/drs/dr10xx.cpp b/clang/test/CXX/drs/dr10xx.cpp index f30ed1cb3e49..77c59078414c 100644 --- a/clang/test/CXX/drs/dr10xx.cpp +++ b/clang/test/CXX/drs/dr10xx.cpp @@ -1,9 +1,9 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors namespace std { __extension__ typedef __SIZE_TYPE__ size_t; @@ -18,30 +18,40 @@ namespace dr1004 { // dr1004: 5 template struct A {}; template struct B1 {}; template class> struct B2 {}; - template void f(); // expected-note {{[with X = dr1004::A]}} - template class X> void f(); // expected-note {{[with X = dr1004::A]}} - template class X> void g(); // expected-note {{[with X = dr1004::A]}} - template void g(); // expected-note {{[with X = dr1004::A]}} + template void f(); // #dr1004-f-1 + template class X> void f(); // #dr1004-f-2 + template class X> void g(); // #dr1004-g-1 + template void g(); // #dr1004-g-2 struct C : A { B1 b1a; B2 b2a; void h() { - f(); // expected-error {{ambiguous}} - g(); // expected-error {{ambiguous}} + f(); + // expected-error@-1 {{call to 'f' is ambiguous}} + // expected-note@#dr1004-f-1 {{candidate function [with X = dr1004::A]}} + // expected-note@#dr1004-f-2 {{candidate function [with X = dr1004::A]}} + g(); + // expected-error@-1 {{call to 'g' is ambiguous}} + // expected-note@#dr1004-g-1 {{candidate function [with X = dr1004::A]}} + // expected-note@#dr1004-g-2 {{candidate function [with X = dr1004::A]}} } }; // This example (from the standard) is actually ill-formed, because // name lookup of "T::template A" names the constructor. - template class U = T::template A> struct Third { }; // expected-error {{is a constructor name}} - Third > t; // expected-note {{in instantiation of default argument}} + template class U = T::template A> struct Third { }; + // expected-error@-1 {{is a constructor name}} + // expected-note@#dr1004-t {{in instantiation of default argument}} + Third > t; // #dr1004-t } namespace dr1042 { // dr1042: 3.5 #if __cplusplus >= 201402L // C++14 added an attribute that we can test the semantics of. - using foo [[deprecated]] = int; // expected-note {{'foo' has been explicitly marked deprecated here}} - foo f = 12; // expected-warning {{'foo' is deprecated}} + using foo [[deprecated]] = int; // #dr1042-using + foo f = 12; + // since-cxx14-warning@-1 {{'foo' is deprecated}} + // since-cxx14-note@#dr1042-using {{'foo' has been explicitly marked deprecated here}} #elif __cplusplus >= 201103L // C++11 did not have any attributes that could be applied to an alias // declaration, so the best we can test is that we accept an empty attribute @@ -76,7 +86,8 @@ namespace dr1054 { // dr1054: no // which copy-initializes a temporary from 'a'. Therefore this is // ill-formed because A does not have a volatile copy constructor. // (We might want to track this aspect under dr1383 instead?) - a; // expected-warning {{assign into a variable to force a volatile load}} + a; + // expected-warning@-1 {{expression result unused; assign into a variable to force a volatile load}} } } diff --git a/clang/test/CXX/drs/dr11xx.cpp b/clang/test/CXX/drs/dr11xx.cpp index 23756ff1927e..86e726ae8c74 100644 --- a/clang/test/CXX/drs/dr11xx.cpp +++ b/clang/test/CXX/drs/dr11xx.cpp @@ -1,30 +1,30 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2a %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2a %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors -namespace dr1111 { // dr1111: yes +namespace dr1111 { // dr1111: 3.2 namespace example1 { -template struct set; +template struct set; // #dr1111-struct-set struct X { - template void set(const T &value); + template void set(const T &value); // #dr1111-func-set }; void foo() { X x; -#pragma clang diagnostic push -#if __cplusplus < 201103L -#pragma clang diagnostic ignored "-Wambiguous-member-template" -#endif + // FIXME: should we backport C++11 behavior? x.set(3.2); -#pragma clang diagnostic pop + // cxx98-error@-1 {{lookup of 'set' in member access expression is ambiguous; using member of 'X'}} + // cxx98-note@#dr1111-func-set {{lookup in the object type 'X' refers here}} + // cxx98-note@#dr1111-struct-set {{lookup from the current scope refers here}} } struct Y {}; void bar() { Y y; - y.set(3.2); // expected-error {{no member named 'set' in 'dr1111::example1::Y'}} + y.set(3.2); + // expected-error@-1 {{no member named 'set' in 'dr1111::example1::Y'}} } } // namespace example1 @@ -46,8 +46,10 @@ void baz() { namespace dr1113 { // dr1113: partial namespace named { - extern int a; // expected-note {{previous}} - static int a; // expected-error {{static declaration of 'a' follows non-static}} + extern int a; // #dr1113-a + static int a; + // expected-error@-1 {{static declaration of 'a' follows non-static}} + // expected-note@#dr1113-a {{previous declaration is here}} } namespace { extern int a; diff --git a/clang/test/CXX/drs/dr12xx.cpp b/clang/test/CXX/drs/dr12xx.cpp index c23a515ba56c..adf7f56711c4 100644 --- a/clang/test/CXX/drs/dr12xx.cpp +++ b/clang/test/CXX/drs/dr12xx.cpp @@ -1,9 +1,9 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected,cxx98-14,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx98-14,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx17,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx17,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx17,since-cxx14,since-cxx11,since-cxx23 -fexceptions -fcxx-exceptions -pedantic-errors // dr1200: na @@ -40,28 +40,33 @@ struct S { S* operator()(); int N; int M; -#if __cplusplus > 202002L +#if __cplusplus >= 202302L template static constexpr auto V = 0; void f(char); void f(int); void mem(S s) { - auto(s)()->M; //expected-warning {{expression result unused}} - auto(s)()->V; //expected-warning {{expression result unused}} + auto(s)()->M; + // since-cxx23-warning@-1 {{expression result unused}} + auto(s)()->V; + // since-cxx23-warning@-1 {{expression result unused}} auto(s)()->f(0); } #endif }; void f(S s) { { -#if __cplusplus > 202002L - auto(s)()->N; //expected-warning {{expression result unused}} +#if __cplusplus >= 202302L + auto(s)()->N; + //since-cxx23-warning@-1 {{expression result unused}} #endif auto(s)()->M; } { - S(s)()->N; //expected-warning {{expression result unused}} - S(s)()->M; //expected-warning {{expression result unused}} + S(s)()->N; + // since-cxx11-warning@-1 {{expression result unused}} + S(s)()->M; + // since-cxx11-warning@-1 {{expression result unused}} } } @@ -74,23 +79,27 @@ void g() { A a(B ()->C); A b(auto ()->C); static_assert(sizeof(B ()->C[1] == sizeof(int)), ""); - sizeof(auto () -> C[1]); // expected-error{{function cannot return array type 'C[1]'}} + sizeof(auto () -> C[1]); + // since-cxx11-error@-1 {{function cannot return array type 'C[1]' (aka 'dr1223::BB[1]')}} } } #endif #if __cplusplus >= 201103L -namespace dr1227 { // dr1227: yes -template struct A { using X = typename T::X; }; // expected-error {{type 'int' cannot be used prior to '::' because it has no members}} +namespace dr1227 { // dr1227: 3.0 +template struct A { using X = typename T::X; }; +// since-cxx11-error@-1 {{type 'int' cannot be used prior to '::' because it has no members}} +// since-cxx11-note@#dr1227-g {{in instantiation of template class 'dr1227::A' requested here}} +// since-cxx11-note@#dr1227-g-int {{while substituting explicitly-specified template arguments into function template 'g'}} template typename T::X f(typename A::X); template void f(...) { } -template auto g(typename A::X) -> typename T::X; // expected-note {{in instantiation of template class 'dr1227::A' requested here}} +template auto g(typename A::X) -> typename T::X; // #dr1227-g template void g(...) { } void h() { f(0); // OK, substituting return type causes deduction to fail - g(0); // expected-note {{while substituting explicitly-specified template arguments into function template 'g'}} + g(0); // #dr1227-g-int } } #endif @@ -109,15 +118,21 @@ struct Derived : Base { namespace dr1265 { // dr1265: 5 #if __cplusplus >= 201103L - auto a = 0, b() -> int; // expected-error {{declaration with trailing return type must be the only declaration in its group}} - auto b() -> int, d = 0; // expected-error {{declaration with trailing return type must be the only declaration in its group}} - auto e() -> int, f() -> int; // expected-error {{declaration with trailing return type must be the only declaration in its group}} + auto a = 0, b() -> int; + // since-cxx11-error@-1 {{declaration with trailing return type must be the only declaration in its group}} + auto b() -> int, d = 0; + // since-cxx11-error@-1 {{declaration with trailing return type must be the only declaration in its group}} + auto e() -> int, f() -> int; + // since-cxx11-error@-1 {{declaration with trailing return type must be the only declaration in its group}} #endif #if __cplusplus >= 201402L - auto g(), h = 0; // expected-error {{function with deduced return type must be the only declaration in its group}} - auto i = 0, j(); // expected-error {{function with deduced return type must be the only declaration in its group}} - auto k(), l(); // expected-error {{function with deduced return type must be the only declaration in its group}} + auto g(), h = 0; + // since-cxx14-error@-1 {{function with deduced return type must be the only declaration in its group}} + auto i = 0, j(); + // since-cxx14-error@-1 {{function with deduced return type must be the only declaration in its group}} + auto k(), l(); + // since-cxx14-error@-1 {{function with deduced return type must be the only declaration in its group}} #endif } @@ -130,16 +145,16 @@ namespace dr1295 { // dr1295: 4 X x = {1}; - unsigned const &r1 = static_cast(x).bitfield; // expected-error 0-1{{C++11}} - unsigned const &r2 = static_cast(x.bitfield); // expected-error 0-1{{C++11}} + unsigned const &r1 = static_cast(x).bitfield; + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + unsigned const &r2 = static_cast(x.bitfield); + // cxx98-error@-1 {{rvalue references are a C++11 extension}} - template struct Y {}; - Y y; -#if __cplusplus <= 201402L - // expected-error@-2 {{does not refer to any declaration}} expected-note@-3 {{here}} -#else - // expected-error@-4 {{refers to subobject}} -#endif + template struct Y {}; // #dr1295-Y + Y y; // #dr1295-y + // cxx98-14-error@-1 {{non-type template argument does not refer to any declaration}} + // cxx98-14-note@#dr1295-Y {{template parameter is declared here}} + // since-cxx17-error@#dr1295-y {{non-type template argument refers to subobject 'x.bitfield'}} #if __cplusplus >= 201103L const unsigned other = 0; diff --git a/clang/test/CXX/drs/dr13xx.cpp b/clang/test/CXX/drs/dr13xx.cpp index 3510695954e2..359c04b3e0f3 100644 --- a/clang/test/CXX/drs/dr13xx.cpp +++ b/clang/test/CXX/drs/dr13xx.cpp @@ -1,7 +1,10 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected,cxx98-14,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11-17,cxx11-14,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx11-17,cxx11-14,since-cxx14,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,cxx11-17,since-cxx14,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors __extension__ typedef __SIZE_TYPE__ size_t; @@ -14,11 +17,13 @@ namespace std { } #if __cplusplus >= 201103L -namespace dr1305 { // dr1305: yes -struct Incomplete; // expected-note {{forward declaration of 'dr1305::Incomplete'}} +namespace dr1305 { // dr1305: 3.0 +struct Incomplete; // #dr1305-Incomplete struct Complete {}; -int incomplete = alignof(Incomplete(&)[]); // expected-error {{invalid application of 'alignof' to an incomplete type 'Incomplete'}} +int incomplete = alignof(Incomplete(&)[]); +// since-cxx11-error@-1 {{invalid application of 'alignof' to an incomplete type 'Incomplete'}} +// since-cxx11-note@#dr1305-Incomplete {{forward declaration of 'dr1305::Incomplete'}} int complete = alignof(Complete(&)[]); } #endif @@ -36,9 +41,11 @@ void caller() { } // namespace dr1307 namespace dr1310 { // dr1310: 5 - struct S {} * sp = new S::S; // expected-error {{qualified reference to 'S' is a constructor name}} + struct S {} * sp = new S::S; + // expected-error@-1 {{qualified reference to 'S' is a constructor name rather than a type in this context}} void f() { - S::S(a); // expected-error {{qualified reference to 'S' is a constructor name}} + S::S(a); + // expected-error@-1 {{qualified reference to 'S' is a constructor name rather than a type in this context}} } struct T { int n; typedef int U; typedef T V; }; int k = T().T::T::n; @@ -64,39 +71,67 @@ namespace dr1310 { // dr1310: 5 template struct W : WBase { typedef int X; int n; }; void w_test() { - W::W w1a; // expected-error {{qualified reference to 'W' is a constructor name}} + W::W w1a; + // expected-error@-1 {{qualified reference to 'W' is a constructor name rather than a type in this context}} W::W::X w1ax; - W::W w1b; // expected-error {{qualified reference to 'W' is a constructor name}} + W::W w1b; + // expected-error@-1 {{qualified reference to 'W' is a constructor name rather than a template name in this context}} W::W::X w1bx; - typename W::W w2a; // expected-error {{qualified reference to 'W' is a constructor name}} expected-error 0-1{{outside of a template}} - typename W::W::X w2ax; // expected-error 0-1{{outside of a template}} - typename W::W w2b; // expected-error {{qualified reference to 'W' is a constructor name}} expected-error 0-1{{outside of a template}} - typename W::W::X w2bx; // expected-error 0-1{{outside of a template}} - W::template W w3; // expected-error {{qualified reference to 'W' is a constructor name}} expected-error 0-1{{outside of a template}} - W::template W::X w3x; // expected-error 0-1{{outside of a template}} - typename W::template W w4; // expected-error {{qualified reference to 'W' is a constructor name}} expected-error 0-2{{outside of a template}} - typename W::template W::X w4x; // expected-error 0-2{{outside of a template}} - - TT::W> tt1; // expected-error {{qualified reference to 'W' is a constructor name}} - TTy::W> tt1a; // expected-error {{qualified reference to 'W' is a constructor name}} - TT::template W> tt2; // expected-error {{qualified reference to 'W' is a constructor name}} expected-error 0-1{{outside of a template}} + typename W::W w2a; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a type in this context, despite preceding 'typename' keyword}} + // cxx98-error@-2 {{'typename' occurs outside of a template}} + typename W::W::X w2ax; + // cxx98-error@-1 {{'typename' occurs outside of a template}} + typename W::W w2b; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'typename' keyword}} + // cxx98-error@-2 {{'typename' occurs outside of a template}} + typename W::W::X w2bx; + // cxx98-error@-1 {{'typename' occurs outside of a template}} + W::template W w3; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'template' keyword}} + // cxx98-error@-2 {{'template' keyword outside of a template}} + W::template W::X w3x; + // cxx98-error@-1 {{'template' keyword outside of a template}} + typename W::template W w4; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'template' keyword}} + // cxx98-error@-2 {{'template' keyword outside of a template}} + // cxx98-error@-3 {{'typename' occurs outside of a template}} + typename W::template W::X w4x; + // cxx98-error@-1 {{'template' keyword outside of a template}} + // cxx98-error@-2 {{'typename' occurs outside of a template}} + + TT::W> tt1; + // expected-error@-1 {{qualified reference to 'W' is a constructor name rather than a type in this context}} + TTy::W> tt1a; + // expected-error@-1 {{qualified reference to 'W' is a constructor name rather than a type in this context}} + TT::template W> tt2; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'template' keyword}} + // cxx98-error@-2 {{'template' keyword outside of a template}} TT::WBase> tt3; TTy::WBase> tt3a; - TT::template WBase> tt4; // expected-error 0-1{{outside of a template}} + TT::template WBase> tt4; + // cxx98-error@-1 {{'template' keyword outside of a template}} W w; (void)w.W::W::n; (void)w.W::W::n; (void)w.W::W::n; - (void)w.W::template W::n; // expected-error 0-1{{outside of a template}} + (void)w.W::template W::n; + // cxx98-error@-1 {{'template' keyword outside of a template}} } template void wt_test() { - typename W::W w2a; // expected-error {{qualified reference to 'W' is a constructor name}} - typename W::template W w4; // expected-error {{qualified reference to 'W' is a constructor name}} - TTy tt2; // expected-error {{qualified reference to 'W' is a constructor name}} - TT tt3; // expected-error {{qualified reference to 'W' is a constructor name}} + typename W::W w2a; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a type in this context, despite preceding 'typename' keyword}} + // cxx98-note@#dr1310-W-int {{in instantiation of function template specialization 'dr1310::wt_test >' requested here}} + // since-cxx11-note@#dr1310-W-int {{in instantiation of function template specialization 'dr1310::wt_test>' requested here}} + typename W::template W w4; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'template' keyword}} + TTy tt2; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a type in this context, despite preceding 'typename' keyword}} + TT tt3; + // expected-error@-1 {{ISO C++ specifies that qualified reference to 'W' is a constructor name rather than a template name in this context, despite preceding 'template' keyword}} } template void wt_test_good() { @@ -111,18 +146,19 @@ namespace dr1310 { // dr1310: 5 (void)w.template W::W::n; (void)w.template W::template W::n; } - template void wt_test >(); // expected-note {{instantiation of}} + template void wt_test >(); // #dr1310-W-int template void wt_test_good >(); } namespace dr1315 { // dr1315: partial template struct A {}; - template // expected-note {{non-deducible template parameter 'I'}} - struct A {}; // expected-error {{contains a template parameter that cannot be deduced}} + template struct A {}; + // expected-error@-1 {{class template partial specialization contains a template parameter that cannot be deduced; this partial specialization will never be used}} + // expected-note@-2 {{non-deducible template parameter 'I'}} template struct A {}; template struct B; - template struct B {}; // expected-note {{matches}} + template struct B {}; // #dr1315-B-1 B<1, 2, 3> b1; // Multiple declarations with the same dependent expression are equivalent @@ -131,8 +167,11 @@ namespace dr1315 { // dr1315: partial B<1, 2, 2>::type b2; // Multiple declarations with differing dependent expressions are unordered. - template struct B {}; // expected-note {{matches}} - B<1, 2, 4> b3; // expected-error {{ambiguous}} + template struct B {}; // #dr1315-B-2 + B<1, 2, 4> b3; + // expected-error@-1 {{ambiguous partial specializations of 'B<1, 2, 4>'}} + // expected-note@#dr1315-B-1 {{partial specialization matches [with I = 1, K = 4]}} + // expected-note@#dr1315-B-2 {{partial specialization matches [with I = 1, K = 4]}} // FIXME: Under dr1315, this is perhaps valid, but that is not clear: this // fails the "more specialized than the primary template" test because the @@ -146,7 +185,9 @@ namespace dr1315 { // dr1315: partial namespace dr1330 { // dr1330: 4 c++11 // exception-specifications are parsed in a context where the class is complete. struct A { - void f() throw(T) {} // expected-error 0-1{{C++17}} expected-note 0-1{{noexcept}} + void f() throw(T) {} + // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@-2 {{use 'noexcept(false)' instead}} struct T {}; #if __cplusplus >= 201103L @@ -156,8 +197,12 @@ namespace dr1330 { // dr1330: 4 c++11 #endif }; - void (A::*af1)() throw(A::T) = &A::f; // expected-error 0-1{{C++17}} expected-note 0-1{{noexcept}} - void (A::*af2)() throw() = &A::f; // expected-error-re {{{{not superset|different exception spec}}}} + void (A::*af1)() throw(A::T) = &A::f; + // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@-2 {{use 'noexcept(false)' instead}} + void (A::*af2)() throw() = &A::f; + // cxx98-14-error@-1 {{target exception specification is not superset of source}} + // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (dr1330::A::*)() throw()' with an rvalue of type 'void (dr1330::A::*)() throw(T)': different exception specifications}} #if __cplusplus >= 201103L static_assert(noexcept(A().g()), ""); @@ -166,7 +211,9 @@ namespace dr1330 { // dr1330: 4 c++11 // Likewise, they're instantiated separately from an enclosing class template. template struct B { - void f() throw(T, typename U::type) {} // expected-error 0-1{{C++17}} expected-note 0-1{{noexcept}} + void f() throw(T, typename U::type) {} + // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@-2 {{use 'noexcept(false)' instead}} struct T {}; #if __cplusplus >= 201103L @@ -183,9 +230,6 @@ namespace dr1330 { // dr1330: 4 c++11 static const int value = true; }; - void (B

::*bpf1)() throw(B

::T, int) = &B

::f; // expected-error 0-1{{C++17}} expected-note 0-1{{noexcept}} -#if __cplusplus < 201103L - // expected-error@-2 {{not superset}} // FIXME: We only delay instantiation in C++11 onwards. In C++98, something // weird happens: instantiation of B

fails because it references T before // it's instantiated, but the diagnostic is suppressed in @@ -193,20 +237,20 @@ namespace dr1330 { // dr1330: 4 c++11 // obviously a bad way to react to this situation; we should still producing // the "T has not yet been instantiated" error here, rather than giving // confusing errors later on. -#endif - void (B

::*bpf2)() throw(int) = &B

::f; // expected-error 0-1{{C++17}} expected-note 0-1{{noexcept}} -#if __cplusplus <= 201402L - // expected-error@-2 {{not superset}} -#else - // expected-warning@-4 {{not superset}} -#endif + void (B

::*bpf1)() throw(B

::T, int) = &B

::f; + // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@-2 {{use 'noexcept(false)' instead}} + // cxx98-error@-3 {{target exception specification is not superset of source}} + + void (B

::*bpf2)() throw(int) = &B

::f; + // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@-2 {{use 'noexcept(false)' instead}} + // cxx98-14-error@-3 {{target exception specification is not superset of source}} + // since-cxx17-warning@-4 {{target exception specification is not superset of source}} void (B

::*bpf3)() = &B

::f; void (B

::*bpf4)() throw() = &B

::f; -#if __cplusplus <= 201402L - // expected-error@-2 {{not superset}} -#else - // expected-error@-4 {{different exception specifications}} -#endif + // cxx98-14-error@-1 {{target exception specification is not superset of source}} + // since-cxx17-error@-2 {{cannot initialize a variable of type 'void (B

::*)() throw()' with an rvalue of type 'void (dr1330::B::*)() throw(T, typename P::type)': different exception specifications}} #if __cplusplus >= 201103L static_assert(noexcept(B

{ -#if __cplusplus < 201103L +#if __cplusplus == 199711L static const int n = 0; #else static const int n = *P; @@ -40,11 +40,13 @@ namespace dr2100 { // dr2100: 12 template struct B { static const int n = 1; int f() { - return Y::declared_later; // expected-error {{no member named 'declared_later'}} + return Y::declared_later; + // expected-error@-1 {{no member named 'declared_later' in 'dr2100::Y<1>'}} } int g() { static const int n = 2; - return Y::declared_later; // expected-error {{no member named 'declared_later'}} + return Y::declared_later; + // expected-error@-1 {{no member named 'declared_later' in 'dr2100::Y<2>'}} } }; template struct Y { @@ -55,10 +57,12 @@ namespace dr2100 { // dr2100: 12 namespace dr2103 { // dr2103: yes void f() { int a; - int &r = a; // expected-note {{here}} + int &r = a; // #dr2103-r struct Inner { void f() { - int &s = r; // expected-error {{enclosing function}} + int &s = r; + // expected-error@-1 {{reference to local variable 'r' declared in enclosing function 'dr2103::f'}} + // expected-note@#dr2103-r {{'r' declared here}} (void)s; } }; @@ -84,28 +88,46 @@ namespace dr2126 { // dr2126: 12 A &b = (A &)(const A &)A{1}; // const temporary A &&c = (A &&)(const A &)A{1}; // const temporary - A &&d = {1}; // non-const temporary expected-note {{here}} - const A &e = (A &)(A &&) A{1}; // non-const temporary expected-note {{here}} - A &&f = (A &&)(A &&) A{1}; // non-const temporary expected-note {{here}} + A &&d = {1}; // non-const temporary #dr21260-d + const A &e = (A &)(A &&) A{1}; // non-const temporary #dr21260-e + A &&f = (A &&)(A &&) A{1}; // non-const temporary #dr21260-f constexpr const A &g = {1}; // const temporary - constexpr A &&h = {1}; // non-const temporary expected-note {{here}} + constexpr A &&h = {1}; // non-const temporary #dr21260-h struct B { const A &a; }; - B i = {{1}}; // extending decl not usable in constant expr expected-note {{here}} - const B j = {{1}}; // extending decl not usable in constant expr expected-note {{here}} + B i = {{1}}; // extending decl not usable in constant expr #dr21260-i + const B j = {{1}}; // extending decl not usable in constant expr #dr21260-j constexpr B k = {{1}}; // extending decl usable in constant expr static_assert(a.n == 1, ""); static_assert(b.n == 1, ""); static_assert(c.n == 1, ""); - static_assert(d.n == 1, ""); // expected-error {{constant}} expected-note {{read of temporary}} - static_assert(e.n == 1, ""); // expected-error {{constant}} expected-note {{read of temporary}} - static_assert(f.n == 1, ""); // expected-error {{constant}} expected-note {{read of temporary}} + static_assert(d.n == 1, ""); + // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}} + // since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}} + // since-cxx11-note@#dr21260-d {{temporary created here}} + static_assert(e.n == 1, ""); + // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}} + // since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}} + // since-cxx11-note@#dr21260-e {{temporary created here}} + static_assert(f.n == 1, ""); + // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}} + // since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}} + // since-cxx11-note@#dr21260-f {{temporary created here}} static_assert(g.n == 1, ""); - static_assert(h.n == 1, ""); // expected-error {{constant}} expected-note {{read of temporary}} - static_assert(i.a.n == 1, ""); // expected-error {{constant}} expected-note {{read of non-constexpr variable}} - static_assert(j.a.n == 1, ""); // expected-error {{constant}} expected-note {{read of temporary}} + static_assert(h.n == 1, ""); + // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}} + // since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}} + // since-cxx11-note@#dr21260-h {{temporary created here}} + static_assert(i.a.n == 1, ""); + // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}} + // since-cxx11-note@-2 {{read of non-constexpr variable 'i' is not allowed in a constant expression}} + // since-cxx11-note@#dr21260-i {{declared here}} + static_assert(j.a.n == 1, ""); + // since-cxx11-error@-1 {{static assertion expression is not an integral constant expression}} + // since-cxx11-note@-2 {{read of temporary is not allowed in a constant expression outside the expression that created the temporary}} + // since-cxx11-note@#dr21260-j {{temporary created here}} static_assert(k.a.n == 1, ""); #endif } @@ -128,19 +150,27 @@ struct B{}; void foo() { struct A *b = (1 == 1) ? new struct A : new struct A; - struct S *a = (1 == 1) ? new struct S : new struct S; // expected-error 2{{allocation of incomplete type}} // expected-note 2{{forward}} + struct S *a = (1 == 1) ? new struct S : new struct S; + // expected-error@-1 {{allocation of incomplete type 'struct S'}} + // expected-note@-2 {{forward declaration of 'S'}} + // expected-error@-3 {{allocation of incomplete type 'struct S'}} + // expected-note@-4 {{forward declaration of 'S'}} #if __cplusplus >= 201103L A *aa = new struct A{}; B *bb = new struct B{}; - (void)new struct C{}; // expected-error {{allocation of incomplete type }} // expected-note {{forward}} + (void)new struct C{}; + // since-cxx11-error@-1 {{allocation of incomplete type 'struct C'}} + // since-cxx11-note@-2 {{forward declaration of 'C'}} struct A *c = (1 == 1) ? new struct A {} : new struct A {}; - alignof(struct D{}); // expected-error {{cannot be defined in a type specifier}} + alignof(struct D{}); + // since-cxx11-error@-1 {{'D' cannot be defined in a type specifier}} #endif - sizeof(struct E{}); // expected-error {{cannot be defined in a type specifier}} + sizeof(struct E{}); + // expected-error@-1 {{'E' cannot be defined in a type specifier}} } } @@ -149,7 +179,8 @@ namespace dr2157 { // dr2157: 11 #if __cplusplus >= 201103L enum E : int; struct X { - enum dr2157::E : int(); // expected-error {{only allows ':' in member enumeration declaration to introduce a fixed underlying type}} + enum dr2157::E : int(); + // since-cxx11-error@-1 {{ISO C++ only allows ':' in member enumeration declaration to introduce a fixed underlying type, not an anonymous bit-field}} }; #endif } @@ -159,11 +190,13 @@ namespace dr2157 { // dr2157: 11 namespace dr2170 { // dr2170: 9 #if __cplusplus >= 201103L void f() { - constexpr int arr[3] = {1, 2, 3}; // expected-note {{here}} + constexpr int arr[3] = {1, 2, 3}; // #dr2170-arr struct S { int get(int n) { return arr[n]; } - const int &get_ref(int n) { return arr[n]; } // expected-error {{enclosing function}} - // FIXME: expected-warning@-1 {{reference to stack}} + const int &get_ref(int n) { return arr[n]; } + // since-cxx11-warning@-1 {{reference to stack memory associated with local variable 'arr' returned}} FIXME + // since-cxx11-error@-2 {{reference to local variable 'arr' declared in enclosing function 'dr2170::f'}} + // since-cxx11-note@#dr2170-arr {{'arr' declared here}} }; } #endif @@ -198,22 +231,32 @@ static_assert(!__is_trivially_assignable(NonConstCopy &&, NonConstCopy &&), ""); namespace dr2180 { // dr2180: yes class A { - A &operator=(const A &); // expected-note 0-2{{here}} - A &operator=(A &&); // expected-note 0-2{{here}} expected-error 0-1{{extension}} + A &operator=(const A &); // #dr2180-A-copy + A &operator=(A &&); // #dr2180-A-move + // cxx98-error@-1 {{rvalue references are a C++11 extension}} }; - struct B : virtual A { + struct B : virtual A { // #dr2180-B B &operator=(const B &); - B &operator=(B &&); // expected-error 0-1{{extension}} + B &operator=(B &&); + // cxx98-error@-1 {{rvalue references are a C++11 extension}} virtual void foo() = 0; }; -#if __cplusplus < 201103L - B &B::operator=(const B&) = default; // expected-error {{private member}} expected-error {{extension}} expected-note {{here}} - B &B::operator=(B&&) = default; // expected-error {{private member}} expected-error 2{{extension}} expected-note {{here}} -#else - B &B::operator=(const B&) = default; // expected-error {{would delete}} expected-note@-9{{inaccessible copy assignment}} - B &B::operator=(B&&) = default; // expected-error {{would delete}} expected-note@-10{{inaccessible move assignment}} -#endif + B &B::operator=(const B&) = default; // #dr2180-B-copy + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-2 {{'operator=' is a private member of 'dr2180::A'}} + // cxx98-note@-3 {{in defaulted copy assignment operator for 'dr2180::B' first required here}} + // cxx98-note@#dr2180-A-copy {{implicitly declared private here}} + // since-cxx11-error@#dr2180-B-copy {{defaulting this copy assignment operator would delete it after its first declaration}} + // since-cxx11-note@#dr2180-B {{copy assignment operator of 'B' is implicitly deleted because base class 'A' has an inaccessible copy assignment operator}} + B &B::operator=(B&&) = default; // #dr2180-B-move + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + // cxx98-error@-2 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-3 {{'operator=' is a private member of 'dr2180::A'}} + // cxx98-note@-4 {{in defaulted move assignment operator for 'dr2180::B' first required here}} + // cxx98-note@#dr2180-A-move {{implicitly declared private here}} + // since-cxx11-error@#dr2180-B-move {{defaulting this move assignment operator would delete it after its first declaration}} + // since-cxx11-note@#dr2180-B {{move assignment operator of 'B' is implicitly deleted because base class 'A' has an inaccessible move assignment operator}} } namespace dr2199 { // dr2199: 3.8 diff --git a/clang/test/CXX/drs/dr22xx.cpp b/clang/test/CXX/drs/dr22xx.cpp index cd849443b111..19518247b528 100644 --- a/clang/test/CXX/drs/dr22xx.cpp +++ b/clang/test/CXX/drs/dr22xx.cpp @@ -1,14 +1,19 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++1z -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors + #if __cplusplus >= 201103L namespace dr2211 { // dr2211: 8 void f() { int a; - auto f = [a](int a) { (void)a; }; // expected-error {{a lambda parameter cannot shadow an explicitly captured entity}} - // expected-note@-1{{variable 'a' is explicitly captured here}} + auto f = [a](int a) { (void)a; }; + // since-cxx11-error@-1 {{a lambda parameter cannot shadow an explicitly captured entity}} + // since-cxx11-note@-2 {{variable 'a' is explicitly captured here}} auto g = [=](int a) { (void)a; }; } } @@ -24,9 +29,12 @@ struct A; namespace dr2229 { // dr2229: 7 struct AnonBitfieldQualifiers { - const unsigned : 1; // expected-error {{anonymous bit-field cannot have qualifiers}} - volatile unsigned : 1; // expected-error {{anonymous bit-field cannot have qualifiers}} - const volatile unsigned : 1; // expected-error {{anonymous bit-field cannot have qualifiers}} + const unsigned : 1; + // expected-error@-1 {{anonymous bit-field cannot have qualifiers}} + volatile unsigned : 1; + // expected-error@-1 {{anonymous bit-field cannot have qualifiers}} + const volatile unsigned : 1; + // expected-error@-1 {{anonymous bit-field cannot have qualifiers}} unsigned : 1; const unsigned i1 : 1; @@ -98,7 +106,8 @@ namespace MultilevelSpecialization { template void f(int i = 0, int (&... arr)[V]); }; template<> template - void B::f(int i, int (&arr1)[a], int (&arr2)[b]) {} // expected-error {{does not match}} + void B::f(int i, int (&arr1)[a], int (&arr2)[b]) {} + // since-cxx11-error@-1 {{out-of-line definition of 'f' does not match any declaration in 'dr2233::MultilevelSpecialization::B'}} template<> template<> void B::f<1, 1>(int i, int (&arr1a)[1], int (&arr2a)[1]) {} } @@ -134,10 +143,10 @@ struct C { explicit operator D(); } c; B b1(a); const B &b2{a}; // FIXME ill-formed const B &b3(a); -// expected-error@-1 {{no viable conversion from 'struct A' to 'const B'}} -// expected-note@#dr2267-struct-B {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'struct A' to 'const B &' for 1st argument}} -// expected-note@#dr2267-struct-B {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'struct A' to 'B &&' for 1st argument}} -// expected-note@#dr2267-struct-B {{explicit constructor is not a candidate}} +// since-cxx11-error@-1 {{no viable conversion from 'struct A' to 'const B'}} +// since-cxx11-note@#dr2267-struct-B {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'struct A' to 'const B &' for 1st argument}} +// since-cxx11-note@#dr2267-struct-B {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'struct A' to 'B &&' for 1st argument}} +// since-cxx11-note@#dr2267-struct-B {{explicit constructor is not a candidate}} D d1(c); const D &d2{c}; // FIXME ill-formed diff --git a/clang/test/CXX/drs/dr2354.cpp b/clang/test/CXX/drs/dr2354.cpp deleted file mode 100644 index 3efb0ba55566..000000000000 --- a/clang/test/CXX/drs/dr2354.cpp +++ /dev/null @@ -1,10 +0,0 @@ -// RUN: %clang_cc1 -x c++ -verify %s - -// dr2354: 15 - -namespace DR2354 { - -enum alignas(64) A {}; // expected-error {{'alignas' attribute cannot be applied to an enumeration}} -enum struct alignas(64) B {}; // expected-error {{'alignas' attribute cannot be applied to an enumeration}} - -} // namespace DR2354 diff --git a/clang/test/CXX/drs/dr2390.cpp b/clang/test/CXX/drs/dr2390.cpp index d8ab1e9a1b38..3931365b568c 100644 --- a/clang/test/CXX/drs/dr2390.cpp +++ b/clang/test/CXX/drs/dr2390.cpp @@ -1,6 +1,6 @@ // RUN: %clang_cc1 -E -P %s -o - | FileCheck %s -// dr2390: yes +// dr2390: 14 namespace PR48462 { // Test that macro expansion of the builtin argument works. diff --git a/clang/test/CXX/drs/dr23xx.cpp b/clang/test/CXX/drs/dr23xx.cpp index 6cb10067739f..9ced61d2aae3 100644 --- a/clang/test/CXX/drs/dr23xx.cpp +++ b/clang/test/CXX/drs/dr23xx.cpp @@ -1,9 +1,10 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++98 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,since-cxx11,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx11,since-cxx14,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11,since-cxx14,since-cxx17,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11,since-cxx14,since-cxx17,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx11,since-cxx14,since-cxx17,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s #if __cplusplus >= 201103L namespace dr2303 { // dr2303: 12 @@ -14,8 +15,14 @@ struct A<> {}; template struct A : A {}; struct B : A {}; -struct C : A, A {}; // expected-warning {{direct base 'A' is inaccessible}} -struct D : A, A {}; // expected-warning {{direct base 'A' is inaccessible}} +struct C : A, A {}; +/* since-cxx11-warning@-1 {{direct base 'A' is inaccessible due to ambiguity: + struct dr2303::C -> A -> A + struct dr2303::C -> A}} */ +struct D : A, A {}; +/* since-cxx11-warning@-1 {{direct base 'A' is inaccessible due to ambiguity: + struct dr2303::D -> A + struct dr2303::D -> A -> A}} */ struct E : A {}; struct F : B, E {}; @@ -32,7 +39,10 @@ void g() { f2(&b); f(C{}); f(D{}); - f(F{}); // expected-error {{ambiguous conversion from derived class}} + f(F{}); + /* since-cxx11-error@-1 {{ambiguous conversion from derived class 'const F' to base class 'const A': + struct dr2303::F -> B -> A + struct dr2303::F -> E -> A}} */ } } //namespace dr2303 #endif @@ -65,8 +75,10 @@ namespace dr2352 { // dr2352: 10 int *const *const &f2() { return p; } int **const &f3() { return p; } - const int **const &f4() { return p; } // expected-error {{reference to type 'const int **const' could not bind to an lvalue of type 'int **'}} - const int *const *&f5() { return p; } // expected-error {{binding reference of type 'const int *const *' to value of type 'int **' not permitted due to incompatible qualifiers}} + const int **const &f4() { return p; } + // expected-error@-1 {{reference to type 'const int **const' could not bind to an lvalue of type 'int **'}} + const int *const *&f5() { return p; } + // expected-error@-1 {{binding reference of type 'const int *const *' to value of type 'int **' not permitted due to incompatible qualifiers}} // FIXME: We permit this as a speculative defect resolution, allowing // qualification conversions when forming a glvalue conditional expression. @@ -76,7 +88,8 @@ namespace dr2352 { // dr2352: 10 // FIXME: Should we compute the composite pointer type here and produce an // lvalue of type 'const int *const * const'? const int * const * r; - void *y = &(true ? p : r); // expected-error {{rvalue of type 'const int *const *'}} + void *y = &(true ? p : r); + // expected-error@-1 {{rvalue of type 'const int *const *'}} // FIXME: We order these as a speculative defect resolution. void f(const int * const * const &r); @@ -124,12 +137,22 @@ namespace dr2353 { // dr2353: 9 #pragma clang __debug dump not_use_2 } +namespace dr2354 { // dr2354: 15 +#if __cplusplus >= 201103L +enum alignas(64) A {}; +// since-cxx11-error@-1 {{'alignas' attribute cannot be applied to an enumeration}} +enum struct alignas(64) B {}; +// since-cxx11-error@-1 {{'alignas' attribute cannot be applied to an enumeration}} +#endif +} // namespace dr2354 + #if __cplusplus >= 201402L namespace dr2358 { // dr2358: 16 void f2() { int i = 1; void g1(int = [xxx=1] { return xxx; }()); // OK - void g2(int = [xxx=i] { return xxx; }()); // expected-error {{default argument references local variable 'i' of enclosing function}} + void g2(int = [xxx=i] { return xxx; }()); + // since-cxx14-error@-1 {{default argument references local variable 'i' of enclosing function}} } } #endif @@ -148,7 +171,7 @@ class C { }; } // namespace dr2370 -#if __cplusplus >= 201707L +#if __cplusplus >= 201702L // Otherwise, if the qualified-id std::tuple_size names a complete class // type **with a member value**, the expression std::tuple_size::value shall // be a well-formed integral constant expression @@ -165,7 +188,8 @@ template <> struct std::tuple_size { } // namespace std namespace dr2386 { void no_value() { auto [x, y] = Bad1(); } -void wrong_value() { auto [x, y] = Bad2(); } // expected-error {{decomposes into 42 elements}} +void wrong_value() { auto [x, y] = Bad2(); } +// since-cxx17-error@-1 {{type 'Bad2' decomposes into 42 elements, but only 2 names were provided}} } // namespace dr2386 #endif @@ -177,7 +201,8 @@ namespace dr2387 { // dr2387: 9 extern template int a<0>; // ok template static int b = 0; - extern template int b<0>; // expected-error {{internal linkage}} + extern template int b<0>; + // since-cxx14-error@-1 {{explicit instantiation declaration of 'b<0>' with internal linkage}} template const int c = 0; extern template const int c<0>; // ok, has external linkage despite 'const' diff --git a/clang/test/CXX/drs/dr2406.cpp b/clang/test/CXX/drs/dr2406.cpp deleted file mode 100644 index 7ea0870fb70b..000000000000 --- a/clang/test/CXX/drs/dr2406.cpp +++ /dev/null @@ -1,30 +0,0 @@ -// RUN: %clang_cc1 -x c++ %s -verify - -// dr2406: yes - -void fallthrough(int n) { - void g(), h(), i(); - switch (n) { - case 1: - case 2: - g(); - [[fallthrough]]; - case 3: // warning on fallthrough discouraged - do { - [[fallthrough]]; // expected-error {{fallthrough annotation does not directly precede switch label}} - } while (false); - case 6: - do { - [[fallthrough]]; // expected-error {{fallthrough annotation does not directly precede switch label}} - } while (n); - case 7: - while (false) { - [[fallthrough]]; // expected-error {{fallthrough annotation does not directly precede switch label}} - } - case 5: - h(); - case 4: // implementation may warn on fallthrough - i(); - [[fallthrough]]; // expected-error {{fallthrough annotation does not directly precede switch label}} - } -} diff --git a/clang/test/CXX/drs/dr24xx.cpp b/clang/test/CXX/drs/dr24xx.cpp index 3fd8539be53d..b34ceb420788 100644 --- a/clang/test/CXX/drs/dr24xx.cpp +++ b/clang/test/CXX/drs/dr24xx.cpp @@ -1,9 +1,52 @@ -// RUN: %clang_cc1 -std=c++20 %s -verify -// RUN: %clang_cc1 -std=c++23 %s -verify +// RUN: %clang_cc1 -std=c++98 %s -verify=expected +// RUN: %clang_cc1 -std=c++11 %s -verify=expected +// RUN: %clang_cc1 -std=c++14 %s -verify=expected +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx17 +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx17 +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx17 +// RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx17 + +#if __cplusplus <= 201402L // expected-no-diagnostics +#endif + +namespace dr2406 { // dr2406: 5 +#if __cplusplus >= 201703L +void fallthrough(int n) { + void g(), h(), i(); + switch (n) { + case 1: + case 2: + g(); + [[fallthrough]]; + case 3: // warning on fallthrough discouraged + do { + [[fallthrough]]; + // since-cxx17-error@-1 {{fallthrough annotation does not directly precede switch label}} + } while (false); + case 6: + do { + [[fallthrough]]; + // since-cxx17-error@-1 {{fallthrough annotation does not directly precede switch label}} + } while (n); + case 7: + while (false) { + [[fallthrough]]; + // since-cxx17-error@-1 {{fallthrough annotation does not directly precede switch label}} + } + case 5: + h(); + case 4: // implementation may warn on fallthrough + i(); + [[fallthrough]]; + // since-cxx17-error@-1 {{fallthrough annotation does not directly precede switch label}} + } +} +#endif +} namespace dr2450 { // dr2450: 18 drafting -#if __cplusplus > 202002L +#if __cplusplus >= 202302L struct S {int a;}; template void f(){} @@ -17,7 +60,7 @@ f<{.a= 0}>(); } namespace dr2459 { // dr2459: 18 drafting -#if __cplusplus > 202002L +#if __cplusplus >= 202302L struct A { constexpr A(float) {} }; diff --git a/clang/test/CXX/drs/dr25xx.cpp b/clang/test/CXX/drs/dr25xx.cpp index 3644e4c328b1..8c34b03c22d5 100644 --- a/clang/test/CXX/drs/dr25xx.cpp +++ b/clang/test/CXX/drs/dr25xx.cpp @@ -1,23 +1,23 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors - -#if __cplusplus < 201103L +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20,since-cxx23 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20,since-cxx23 -fexceptions -fcxx-exceptions -pedantic-errors + +#if __cplusplus == 199711L // expected-no-diagnostics #endif -namespace dr2516 { // dr2516: yes +namespace dr2516 { // dr2516: 3.0 // NB: reusing 1482 test #if __cplusplus >= 201103L template struct S { typedef char I; }; enum E2 : S::I { e }; -// expected-error@-1 {{use of undeclared identifier 'E2'}} +// since-cxx11-error@-1 {{use of undeclared identifier 'E2'}} #endif } // namespace dr2516 @@ -27,24 +27,24 @@ namespace dr2518 { // dr2518: 17 template void f(T t) { if constexpr (sizeof(T) != sizeof(int)) { -#if __cplusplus < 201703L -// expected-error@-2 {{constexpr if is a C++17 extension}} -#endif - static_assert(false, "must be int-sized"); // expected-error {{must be int-size}} + // cxx11-14-error@-1 {{constexpr if is a C++17 extension}} + static_assert(false, "must be int-sized"); + // since-cxx11-error@-1 {{static assertion failed: must be int-sized}} + // since-cxx11-note@#dr2518-f-c {{in instantiation of function template specialization 'dr2518::f' requested here}} } } void g(char c) { f(0); - f(c); // expected-note {{requested here}} + f(c); // #dr2518-f-c } template struct S { - static_assert(false); // expected-error {{static assertion failed}} -#if __cplusplus < 201703L -// expected-error@-2 {{'static_assert' with no message is a C++17 extension}} -#endif + static_assert(false); + // cxx11-14-error@-1 {{'static_assert' with no message is a C++17 extension}} + // since-cxx11-error@-2 {{static assertion failed}} + // since-cxx11-note@#dr2518-S-double {{in instantiation of template class 'dr2518::S' requested here}} }; template <> @@ -56,7 +56,7 @@ struct S {}; int test_specialization() { S s1; S s2; - S s3; // expected-note {{in instantiation of template class 'dr2518::S' requested here}} + S s3; // #dr2518-S-double } #endif @@ -67,16 +67,16 @@ namespace dr2521 { // dr2521: 17 #pragma clang diagnostic push #pragma clang diagnostic warning "-Wdeprecated-literal-operator" long double operator"" _\u03C0___(long double); -// expected-warning@-1 {{identifier '_π___' preceded by whitespace in a literal operator declaration is deprecated}} -// expected-warning@-2 {{user-defined literal suffixes containing '__' are reserved}} +// since-cxx11-warning@-1 {{identifier '_π___' preceded by whitespace in a literal operator declaration is deprecated}} +// since-cxx11-warning@-2 {{user-defined literal suffixes containing '__' are reserved}} template decltype(sizeof 0) operator"" _div(); -// expected-warning@-1 {{identifier '_div' preceded by whitespace in a literal operator declaration is deprecated}} +// since-cxx11-warning@-1 {{identifier '_div' preceded by whitespace in a literal operator declaration is deprecated}} using ::dr2521::operator"" _\u03C0___; using ::dr2521::operator""_div; -// expected-warning@-2 {{identifier '_π___' preceded by whitespace in a literal operator declaration is deprecated}} +// since-cxx11-warning@-2 {{identifier '_π___' preceded by whitespace in a literal operator declaration is deprecated}} #pragma clang diagnostic pop #endif } // namespace dr2521 @@ -85,12 +85,16 @@ using ::dr2521::operator""_div; #if __cplusplus >= 202302L namespace dr2553 { // dr2553: 18 struct B { - virtual void f(this B&); // expected-error {{an explicit object parameter cannot appear in a virtual function}} - static void f(this B&); // expected-error {{an explicit object parameter cannot appear in a static function}} - virtual void g(); // expected-note {{here}} + virtual void f(this B&); + // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}} + static void f(this B&); + // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a static function}} + virtual void g(); // #dr2553-g }; struct D : B { - void g(this D&); // expected-error {{an explicit object parameter cannot appear in a virtual function}} + void g(this D&); + // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}} + // since-cxx23-note@#dr2553-g {{overridden virtual function is here}} }; } @@ -99,19 +103,25 @@ struct D : B { #if __cplusplus >= 202302L namespace dr2554 { // dr2554: 18 review struct B { - virtual void f(); // expected-note 3{{here}} + virtual void f(); // #dr2554-g }; struct D : B { - void f(this D&); // expected-error {{an explicit object parameter cannot appear in a virtual function}} + void f(this D&); + // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}} + // since-cxx23-note@#dr2554-g {{overridden virtual function is here}} }; struct D2 : B { - void f(this B&); // expected-error {{an explicit object parameter cannot appear in a virtual function}} + void f(this B&); + // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}} + // since-cxx23-note@#dr2554-g {{overridden virtual function is here}} }; struct T {}; struct D3 : B { - void f(this T&); // expected-error {{an explicit object parameter cannot appear in a virtual function}} + void f(this T&); + // since-cxx23-error@-1 {{an explicit object parameter cannot appear in a virtual function}} + // since-cxx23-note@#dr2554-g {{overridden virtual function is here}} }; } @@ -153,48 +163,48 @@ namespace dr2565 { // dr2565: 16 static_assert(is_referenceable::value); template - concept TwoParams = requires (T *a, U b){ true;}; // #TPC + concept TwoParams = requires (T *a, U b){ true;}; // #dr2565-TPC template - requires TwoParams // #TPSREQ + requires TwoParams // #dr2565-TPSREQ struct TwoParamsStruct{}; using TPSU = TwoParamsStruct; - // expected-error@-1{{constraints not satisfied for class template 'TwoParamsStruct'}} - // expected-note@#TPSREQ{{because 'TwoParams' evaluated to false}} - // expected-note@#TPC{{because 'b' would be invalid: argument may not have 'void' type}} + // since-cxx20-error@-1 {{constraints not satisfied for class template 'TwoParamsStruct'}} + // since-cxx20-note@#dr2565-TPSREQ {{because 'TwoParams' evaluated to false}} + // since-cxx20-note@#dr2565-TPC {{because 'b' would be invalid: argument may not have 'void' type}} template - concept Variadic = requires (U* ... a, T b){ true;}; // #VC + concept Variadic = requires (U* ... a, T b){ true;}; // #dr2565-VC template - requires Variadic // #VSREQ + requires Variadic // #dr2565-VSREQ struct VariadicStruct{}; using VSU = VariadicStruct; - // expected-error@-1{{constraints not satisfied for class template 'VariadicStruct'}} - // expected-note@#VSREQ{{because 'Variadic' evaluated to false}} - // expected-note@#VC{{because 'b' would be invalid: argument may not have 'void' type}} + // since-cxx20-error@-1 {{constraints not satisfied for class template 'VariadicStruct'}} + // since-cxx20-note@#dr2565-VSREQ {{because 'Variadic' evaluated to false}} + // since-cxx20-note@#dr2565-VC {{because 'b' would be invalid: argument may not have 'void' type}} template - // expected-error@+1 {{unknown type name 'ErrorRequires'}} concept ErrorRequires = requires (ErrorRequires auto x) { + // since-cxx20-error@-1 {{unknown type name 'ErrorRequires'}} x; }; static_assert(ErrorRequires); - // expected-error@-1{{static assertion failed}} - // expected-note@-2{{because substituted constraint expression is ill-formed: constraint depends on a previously diagnosed expression}} + // since-cxx20-error@-1 {{static assertion failed}} + // since-cxx20-note@-2 {{because substituted constraint expression is ill-formed: constraint depends on a previously diagnosed expression}} template - // expected-error@+2 {{unknown type name 'NestedErrorInRequires'}} concept NestedErrorInRequires = requires (T x) { requires requires (NestedErrorInRequires auto y) { + // since-cxx20-error@-1 {{unknown type name 'NestedErrorInRequires'}} y; }; }; static_assert(NestedErrorInRequires); - // expected-error@-1{{static assertion failed}} - // expected-note@-2{{because substituted constraint expression is ill-formed: constraint depends on a previously diagnosed expression}} + // expected-error@-1 {{static assertion failed}} + // expected-note@-2 {{because substituted constraint expression is ill-formed: constraint depends on a previously diagnosed expression}} #endif } diff --git a/clang/test/CXX/drs/dr26xx.cpp b/clang/test/CXX/drs/dr26xx.cpp index 8517cd5872b1..dd4bb1ff6ae2 100644 --- a/clang/test/CXX/drs/dr26xx.cpp +++ b/clang/test/CXX/drs/dr26xx.cpp @@ -1,8 +1,14 @@ -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -// RUN: %clang_cc1 -std=c++2b -triple x86_64-unknown-unknown %s -verify +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,cxx11 +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20 +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20,since-cxx23 +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx20,since-cxx23 -namespace dr2621 { // dr2621: yes +namespace dr2621 { // dr2621: 16 +#if __cplusplus >= 202002L enum class E { a }; namespace One { using E_t = E; @@ -12,33 +18,39 @@ auto v = a; namespace Two { using dr2621::E; int E; // we see this -using enum E; // expected-error {{unknown type name E}} +using enum E; +// since-cxx20-error@-1 {{unknown type name E}} } +#endif } namespace dr2628 { // dr2628: no open // this was reverted for the 16.x release // due to regressions, see the issue for more details: // https://github.com/llvm/llvm-project/issues/60777 - +#if __cplusplus >= 202002L template struct foo { // The expected notes below should be removed when dr2628 is fully implemented again - constexpr foo() requires (!A && !B) = delete; // expected-note {{candidate function [with A = false, B = false]}} #DR2628_CTOR - constexpr foo() requires (A || B) = delete; // expected-note {{candidate function [with A = false, B = false]}} + constexpr foo() requires (!A && !B) = delete; // #dr2628-ctor-1 + constexpr foo() requires (A || B) = delete; // #dr2628-ctor-2 }; void f() { // The FIXME's below should be the expected errors when dr2628 is // fully implemented again. - // FIXME-expected-error {{call to deleted}} - foo fooable; // expected-error {{ambiguous deduction for template arguments of 'foo'}} - // FIXME-expected-note@#DR2628_CTOR {{marked deleted here}} + foo fooable; // #dr2628-fooable + // since-cxx20-error@-1 {{ambiguous deduction for template arguments of 'foo'}} + // since-cxx20-note@#dr2628-ctor-1 {{candidate function [with A = false, B = false]}} + // since-cxx20-note@#dr2628-ctor-2 {{candidate function [with A = false, B = false]}} + // FIXME-since-cxx20-error@#dr2628-fooable {{call to deleted}} + // FIXME-since-cxx20-note@#dr2628-ctor {{marked deleted here}} } - +#endif } namespace dr2631 { // dr2631: 16 +#if __cplusplus >= 202002L constexpr int g(); consteval int f() { return g(); @@ -52,9 +64,11 @@ namespace dr2631 { // dr2631: 16 int test() { return k(); } +#endif } namespace dr2635 { // dr2635: 16 +#if __cplusplus >= 202002L template concept UnaryC = true; template @@ -67,66 +81,79 @@ template T get_T(); void use() { - // expected-error@+1{{decomposition declaration cannot be declared with constrained 'auto'}} UnaryC auto [a, b] = get_S(); - // expected-error@+1{{decomposition declaration cannot be declared with constrained 'auto'}} + // since-cxx20-error@-1 {{decomposition declaration cannot be declared with constrained 'auto'}} BinaryC auto [c, d] = get_S(); + // since-cxx20-error@-1 {{decomposition declaration cannot be declared with constrained 'auto'}} } template void TemplUse() { - // expected-error@+1{{decomposition declaration cannot be declared with constrained 'auto'}} UnaryC auto [a, b] = get_T(); - // expected-error@+1{{decomposition declaration cannot be declared with constrained 'auto'}} + // since-cxx20-error@-1 {{decomposition declaration cannot be declared with constrained 'auto'}} BinaryC auto [c, d] = get_T(); + // since-cxx20-error@-1 {{decomposition declaration cannot be declared with constrained 'auto'}} } +#endif } - // dr2636: na +// dr2636: na namespace dr2640 { // dr2640: 16 -int \N{Λ} = 0; //expected-error {{'Λ' is not a valid Unicode character name}} \ - //expected-error {{expected unqualified-id}} -const char* emoji = "\N{🤡}"; // expected-error {{'🤡' is not a valid Unicode character name}} \ - // expected-note 5{{did you mean}} +int \N{Λ} = 0; +// expected-error@-1 {{'Λ' is not a valid Unicode character name}} +// expected-error@-2 {{expected unqualified-id}} +const char* emoji = "\N{🤡}"; +// expected-error@-1 {{'🤡' is not a valid Unicode character name}} +// expected-note@-2 {{did you mean OX ('🐂' U+1F402)?}} +// expected-note@-3 {{did you mean ANT ('🐜' U+1F41C)?}} +// expected-note@-4 {{did you mean ARC ('⌒' U+2312)?}} +// expected-note@-5 {{did you mean AXE ('🪓' U+1FA93)?}} +// expected-note@-6 {{did you mean BAT ('🦇' U+1F987)?}} #define z(x) 0 #define dr2640_a z( -int x = dr2640_a\N{abc}); // expected-error {{'abc' is not a valid Unicode character name}} -int y = dr2640_a\N{LOTUS}); // expected-error {{character not allowed in an identifier}} \ - // expected-error {{use of undeclared identifier 'dr2640_a🪷'}} \ - // expected-error {{extraneous ')' before ';'}} -} - - // dr2642: na - -namespace dr2644 { // dr2644: yes - -auto z = [a = 42](int a) { // expected-error {{a lambda parameter cannot shadow an explicitly captured entity}} \ - // expected-note {{variable 'a' is explicitly captured here}} +int x = dr2640_a\N{abc}); +// expected-error@-1 {{'abc' is not a valid Unicode character name}} +int y = dr2640_a\N{LOTUS}); +// expected-error@-1 {{character not allowed in an identifier}} +// expected-error@-2 {{use of undeclared identifier 'dr2640_a🪷'}} +// expected-error@-3 {{extraneous ')' before ';'}} +} + +// dr2642: na + +namespace dr2644 { // dr2644: 8 +#if __cplusplus >= 201103L +auto z = [a = 42](int a) { +// cxx11-warning@-1 {{initialized lambda captures are a C++14 extension}} +// since-cxx11-error@-2 {{a lambda parameter cannot shadow an explicitly captured entity}} +// since-cxx11-note@-3 {{variable 'a' is explicitly captured here}} return 1; }; - +#endif } #if __cplusplus >= 202302L -namespace dr2650 { // dr2650: yes +namespace dr2650 { // dr2650: 17 template struct S {}; -template int f(S*); // expected-note {{type 'X' of non-type template parameter is not a structural type}} +template int f(S*); // #dr2650-f class X { int m; }; -int i0 = f(0); //expected-error {{no matching function for call to 'f'}} +int i0 = f(0); +// since-cxx23-error@-1 {{no matching function for call to 'f'}} +// since-cxx23-note@#dr2650-f {{type 'X' of non-type template parameter is not a structural type}} } #endif #if __cplusplus >= 202302L namespace dr2653 { // dr2653: 18 struct Test { void f(this const auto& = Test{}); }; - // expected-error@-1 {{the explicit object parameter cannot have a default argument}} + // since-cxx23-error@-1 {{the explicit object parameter cannot have a default argument}} auto L = [](this const auto& = Test{}){}; - // expected-error@-1 {{the explicit object parameter cannot have a default argument}} + // since-cxx23-error@-1 {{the explicit object parameter cannot have a default argument}} } #endif @@ -141,6 +168,7 @@ void f() { } namespace dr2681 { // dr2681: 17 +#if __cplusplus >= 202002L using size_t = decltype(sizeof(int)); template @@ -152,7 +180,7 @@ struct I { volatile T array[N]; }; template -struct J { // expected-note 3{{candidate}} +struct J { // #dr2681-J unsigned char array[N]; }; @@ -161,15 +189,24 @@ I i = { "def" }; static_assert(__is_same(decltype(h), H)); // Not H static_assert(__is_same(decltype(i), I)); -J j = { "ghi" }; // expected-error {{no viable constructor or deduction guide}} +J j = { "ghi" }; +// since-cxx20-error@-1 {{no viable constructor or deduction guide}} +// since-cxx20-note@#dr2681-J {{candidate template ignored: could not match 'J' against 'const char *'}} +// since-cxx20-note@#dr2681-J {{candidate template ignored: could not match 'const unsigned char' against 'const char'}} +// since-cxx20-note@#dr2681-J {{candidate function template not viable: requires 0 arguments, but 1 was provided}} +#endif } namespace dr2672 { // dr2672: 18 open +#if __cplusplus >= 202002L template -void f(T) requires requires { []() { T::invalid; } (); }; // expected-error{{type 'int' cannot be used prior to '::'}} - // expected-note@-1{{while substituting into a lambda expression here}} - // expected-note@-2{{in instantiation of requirement here}} - // expected-note@-3{{while substituting template arguments into constraint expression here}} +void f(T) requires requires { []() { T::invalid; } (); }; +// since-cxx20-error@-1 {{type 'int' cannot be used prior to '::' because it has no members}} +// since-cxx20-note@-2 {{while substituting into a lambda expression here}} +// since-cxx20-note@-3 {{in instantiation of requirement here}} +// since-cxx20-note@-4 {{while substituting template arguments into constraint expression here}} +// since-cxx20-note@#dr2672-f-0 {{while checking constraint satisfaction for template 'f' required here}} +// since-cxx20-note@#dr2672-f-0 {{in instantiation of function template specialization 'dr2672::f' requested here}} void f(...); template @@ -179,11 +216,12 @@ void bar(T) requires requires { void bar(...); void m() { - f(0); // expected-note {{while checking constraint satisfaction for template 'f' required here}} - // expected-note@-1 {{in instantiation of function template specialization}} + f(0); // #dr2672-f-0 bar(0); } +#endif } + #if __cplusplus >= 202302L namespace dr2687 { // dr2687: 18 struct S{ @@ -193,7 +231,8 @@ struct S{ }; void test() { - (&S::f)(1); // expected-error {{called object type 'void (dr2687::S::*)(int)' is not a function or function pointer}} + (&S::f)(1); + // since-cxx23-error@-1 {{called object type 'void (dr2687::S::*)(int)' is not a function or function pointer}} (&S::g)(1); (&S::h)(S(), 1); } diff --git a/clang/test/CXX/drs/dr27xx.cpp b/clang/test/CXX/drs/dr27xx.cpp index 5c7ce98f878d..4f7d0d6b44a8 100644 --- a/clang/test/CXX/drs/dr27xx.cpp +++ b/clang/test/CXX/drs/dr27xx.cpp @@ -1,6 +1,17 @@ -// RUN: %clang_cc1 -std=c++2c -verify %s +// RUN: %clang_cc1 -std=c++98 -verify=expected %s +// RUN: %clang_cc1 -std=c++11 -verify=expected %s +// RUN: %clang_cc1 -std=c++14 -verify=expected %s +// RUN: %clang_cc1 -std=c++17 -verify=expected %s +// RUN: %clang_cc1 -std=c++20 -verify=expected %s +// RUN: %clang_cc1 -std=c++23 -verify=expected,since-cxx23 %s +// RUN: %clang_cc1 -std=c++2c -verify=expected,since-cxx23,since-cxx26 %s + +#if __cplusplus <= 202002L +// expected-no-diagnostics +#endif namespace dr2789 { // dr2789: 18 open +#if __cplusplus >= 202302L template struct Base { constexpr void g(); // #dr2789-g1 @@ -23,11 +34,12 @@ struct S : Base, Base2 { void test() { S<> s; s.f(); - s.g(); // expected-error {{call to member function 'g' is ambiguous}} - // expected-note@#dr2789-g1 {{candidate function}} - // expected-note@#dr2789-g2 {{candidate function}} + s.g(); + // since-cxx23-error@-1 {{call to member function 'g' is ambiguous}} + // since-cxx23-note@#dr2789-g1 {{candidate function}} + // since-cxx23-note@#dr2789-g2 {{candidate function}} } - +#endif } namespace dr2798 { // dr2798: 17 drafting @@ -49,7 +61,8 @@ struct X { }; consteval X f() { return {}; } -static_assert(false, f().s); // expected-error {{static assertion failed: Hello}} +static_assert(false, f().s); +// since-cxx26-error@-1 {{static assertion failed: Hello}} #endif } // namespace dr2798 diff --git a/clang/test/CXX/drs/dr412.cpp b/clang/test/CXX/drs/dr412.cpp index 27bc7e5cf94e..8ea29135d1df 100644 --- a/clang/test/CXX/drs/dr412.cpp +++ b/clang/test/CXX/drs/dr412.cpp @@ -1,9 +1,12 @@ // RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT="throw()" -DBAD_ALLOC="throw(std::bad_alloc)" // RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= // RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= -// RUN: %clang_cc1 -std=c++1z %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= +// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= +// RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= +// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= +// RUN: %clang_cc1 -std=c++2c %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -DNOEXCEPT=noexcept -DBAD_ALLOC= -// dr412: yes +// dr412: 3.4 // lwg404: yes // lwg2340: yes @@ -11,11 +14,17 @@ __extension__ typedef __SIZE_TYPE__ size_t; namespace std { struct bad_alloc {}; } -inline void* operator new(size_t) BAD_ALLOC; // expected-error {{cannot be declared 'inline'}} -inline void* operator new[](size_t) BAD_ALLOC; // expected-error {{cannot be declared 'inline'}} -inline void operator delete(void*) NOEXCEPT; // expected-error {{cannot be declared 'inline'}} -inline void operator delete[](void*) NOEXCEPT; // expected-error {{cannot be declared 'inline'}} +inline void* operator new(size_t) BAD_ALLOC; +// expected-error@-1 {{replacement function 'operator new' cannot be declared 'inline'}} +inline void* operator new[](size_t) BAD_ALLOC; +// expected-error@-1 {{replacement function 'operator new[]' cannot be declared 'inline'}} +inline void operator delete(void*) NOEXCEPT; +// expected-error@-1 {{replacement function 'operator delete' cannot be declared 'inline'}} +inline void operator delete[](void*) NOEXCEPT; +// expected-error@-1 {{replacement function 'operator delete[]' cannot be declared 'inline'}} #ifdef __cpp_sized_deallocation -inline void operator delete(void*, size_t) NOEXCEPT; // expected-error {{cannot be declared 'inline'}} -inline void operator delete[](void*, size_t) NOEXCEPT; // expected-error {{cannot be declared 'inline'}} +inline void operator delete(void*, size_t) NOEXCEPT; +// expected-error@-1 {{replacement function 'operator delete' cannot be declared 'inline'}} +inline void operator delete[](void*, size_t) NOEXCEPT; +// expected-error@-1 {{replacement function 'operator delete[]' cannot be declared 'inline'}} #endif diff --git a/clang/test/CXX/drs/dr7xx.cpp b/clang/test/CXX/drs/dr7xx.cpp index 11901b80d646..926bff1cc479 100644 --- a/clang/test/CXX/drs/dr7xx.cpp +++ b/clang/test/CXX/drs/dr7xx.cpp @@ -1,37 +1,53 @@ -// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++2a %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++98 %s -verify=expected,cxx98-14,cxx98-11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++11 %s -verify=expected,cxx98-14,cxx98-11,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++14 %s -verify=expected,cxx98-14,since-cxx14,since-cxx11,cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++17 %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -triple %itanium_abi_triple -std=c++2a %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors namespace dr705 { // dr705: yes namespace N { struct S {}; - void f(S); // expected-note {{declared here}} + void f(S); // #dr705-f } void g() { N::S s; f(s); // ok - (f)(s); // expected-error {{use of undeclared}} + (f)(s); + // expected-error@-1 {{use of undeclared identifier 'f'}} + // expected-note@#dr705-f {{'N::f' declared here}} } } namespace dr712 { // dr712: partial void use(int); void f() { - const int a = 0; // expected-note 5{{here}} + const int a = 0; // #dr712-f-a struct X { void g(bool cond) { use(a); use((a)); use(cond ? a : a); - use((cond, a)); // expected-warning 2{{left operand of comma operator has no effect}} FIXME: should only warn once - - (void)a; // FIXME: expected-error {{declared in enclosing}} - (void)(a); // FIXME: expected-error {{declared in enclosing}} - (void)(cond ? a : a); // FIXME: expected-error 2{{declared in enclosing}} - (void)(cond, a); // FIXME: expected-error {{declared in enclosing}} expected-warning {{left operand of comma operator has no effect}} + // FIXME: should only warn once + use((cond, a)); + // expected-warning@-1 {{left operand of comma operator has no effect}} + // expected-warning@-2 {{left operand of comma operator has no effect}} + + (void)a; + // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME + // expected-note@#dr712-f-a {{'a' declared here}} + (void)(a); + // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME + // expected-note@#dr712-f-a {{'a' declared here}} + (void)(cond ? a : a); // #dr712-ternary + // expected-error@#dr712-ternary {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME + // expected-note@#dr712-f-a {{'a' declared here}} + // expected-error@#dr712-ternary {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME + // expected-note@#dr712-f-a {{'a' declared here}} + (void)(cond, a); // #dr712-comma + // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::f'}} FIXME + // expected-note@#dr712-f-a {{'a' declared here}} + // expected-warning@#dr712-comma {{left operand of comma operator has no effect}} } }; } @@ -39,14 +55,18 @@ namespace dr712 { // dr712: partial #if __cplusplus >= 201103L void g() { struct A { int n; }; - constexpr A a = {0}; // expected-note 2{{here}} + constexpr A a = {0}; // #dr712-g-a struct X { void g(bool cond) { use(a.n); use(a.*&A::n); - (void)a.n; // FIXME: expected-error {{declared in enclosing}} - (void)(a.*&A::n); // FIXME: expected-error {{declared in enclosing}} + (void)a.n; + // since-cxx11-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::g'}} FIXME + // since-cxx11-note@#dr712-g-a {{'a' declared here}} + (void)(a.*&A::n); + // since-cxx11-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr712::g'}} FIXME + // since-cxx11-note@#dr712-g-a {{'a' declared here}} } }; } @@ -55,9 +75,10 @@ namespace dr712 { // dr712: partial namespace dr727 { // dr727: partial struct A { - template struct C; // expected-note 6{{here}} - template void f(); // expected-note {{here}} - template static int N; // expected-error 0-1{{C++14}} expected-note 6{{here}} + template struct C; // #dr727-C + template void f(); // #dr727-f + template static int N; // #dr727-N + // cxx98-11-error@-1 {{variable templates are a C++14 extension}} template<> struct C; template<> void f(); @@ -67,19 +88,40 @@ namespace dr727 { // dr727: partial template static int N; struct B { - template<> struct C; // expected-error {{not in class 'A' or an enclosing namespace}} - template<> void f(); // expected-error {{no function template matches}} - template<> static int N; // expected-error {{not in class 'A' or an enclosing namespace}} - - template struct C; // expected-error {{not in class 'A' or an enclosing namespace}} - template static int N; // expected-error {{not in class 'A' or an enclosing namespace}} - - template<> struct A::C; // expected-error {{not in class 'A' or an enclosing namespace}} - template<> void A::f(); // expected-error {{no function template matches}} expected-error {{cannot have a qualified name}} - template<> static int A::N; // expected-error {{not in class 'A' or an enclosing namespace}} expected-error {{cannot have a qualified name}} - - template struct A::C; // expected-error {{not in class 'A' or an enclosing namespace}} - template static int A::N; // expected-error {{not in class 'A' or an enclosing namespace}} expected-error {{cannot have a qualified name}} + template<> struct C; + // expected-error@-1 {{class template specialization of 'C' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-C {{explicitly specialized declaration is here}} + template<> void f(); + // expected-error@-1 {{no function template matches function template specialization 'f'}} + template<> static int N; + // expected-error@-1 {{variable template specialization of 'N' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-N {{explicitly specialized declaration is here}} + + template struct C; + // expected-error@-1 {{class template partial specialization of 'C' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-C {{explicitly specialized declaration is here}} + template static int N; + // expected-error@-1 {{variable template partial specialization of 'N' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-N {{explicitly specialized declaration is here}} + + template<> struct A::C; + // expected-error@-1 {{class template specialization of 'C' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-C {{explicitly specialized declaration is here}} + template<> void A::f(); + // expected-error@-1 {{o function template matches function template specialization 'f'}} + // expected-error@-2 {{non-friend class member 'f' cannot have a qualified name}} + template<> static int A::N; + // expected-error@-1 {{non-friend class member 'N' cannot have a qualified name}} + // expected-error@-2 {{variable template specialization of 'N' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-N {{explicitly specialized declaration is here}} + + template struct A::C; + // expected-error@-1 {{class template partial specialization of 'C' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-C {{explicitly specialized declaration is here}} + template static int A::N; + // expected-error@-1 {{non-friend class member 'N' cannot have a qualified name}} + // expected-error@-2 {{variable template partial specialization of 'N' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-N {{explicitly specialized declaration is here}} }; }; @@ -91,19 +133,36 @@ namespace dr727 { // dr727: partial template int A::N; namespace C { - template<> struct A::C; // expected-error {{not in class 'A' or an enclosing namespace}} - template<> void A::f(); // expected-error {{not in class 'A' or an enclosing namespace}} - template<> int A::N; // expected-error {{not in class 'A' or an enclosing namespace}} - - template struct A::C; // expected-error {{not in class 'A' or an enclosing namespace}} - template int A::N; // expected-error {{not in class 'A' or an enclosing namespace}} + template<> struct A::C; + // expected-error@-1 {{class template specialization of 'C' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-C {{explicitly specialized declaration is here}} + template<> void A::f(); + // expected-error@-1 {{function template specialization of 'f' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-f {{explicitly specialized declaration is here}} + template<> int A::N; + // expected-error@-1 {{variable template specialization of 'N' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-N {{explicitly specialized declaration is here}} + + template struct A::C; + // expected-error@-1 {{class template partial specialization of 'C' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-C {{explicitly specialized declaration is here}} + template int A::N; + // expected-error@-1 {{variable template partial specialization of 'N' not in class 'A' or an enclosing namespace}} + // expected-note@#dr727-N {{explicitly specialized declaration is here}} } template struct D { - template struct C { typename T::error e; }; // expected-error {{no members}} - template void f() { T::error; } // expected-error {{no members}} - template static const int N = T::error; // expected-error {{no members}} expected-error 0-1{{C++14}} + template struct C { typename T::error e; }; + // expected-error@-1 {{type 'float' cannot be used prior to '::' because it has no members}} + // expected-note@#dr727-C-float {{in instantiation of template class 'dr727::D::C' requested here}} + template void f() { T::error; } + // expected-error@-1 {{type 'float' cannot be used prior to '::' because it has no members}} + // expected-note@#dr727-f-float {{in instantiation of function template specialization 'dr727::D::f' requested here}} + template static const int N = T::error; + // cxx98-11-error@-1 {{variable templates are a C++14 extension}} + // expected-error@-2 {{type 'float' cannot be used prior to '::' because it has no members}} + // expected-note@#dr727-N-float {{in instantiation of static data member 'dr727::D::N' requested here}} template<> struct C {}; template<> void f() {} @@ -114,7 +173,8 @@ namespace dr727 { // dr727: partial template struct E { - template<> void f() {} // expected-error {{no candidate function template}} + template<> void f() {} + // expected-error@-1 {{no candidate function template was found for dependent member function template specialization}} }; }; @@ -126,9 +186,9 @@ namespace dr727 { // dr727: partial D::C(); int b = D::N; - D::C(); // expected-note {{instantiation of}} - di.f(); // expected-note {{instantiation of}} - int c = D::N; // expected-note {{instantiation of}} + D::C(); // #dr727-C-float + di.f(); // #dr727-f-float + int c = D::N; // #dr727-N-float } namespace mixed_inner_outer_specialization { @@ -148,28 +208,30 @@ namespace dr727 { // dr727: partial #if __cplusplus >= 201402L template struct B { template static const int u = 1; - template<> static const int u<0> = 2; // expected-note {{here}} + template<> static const int u<0> = 2; // #dr727-u0 // Note that in C++17 onwards, these are implicitly inline, and so the // initializer of v<0> is not instantiated with the declaration. In // C++14, v<0> is a non-defining declaration and its initializer is // instantiated with the class. template static constexpr int v = 1; - template<> static constexpr int v<0> = 2; // #v0 + template<> static constexpr int v<0> = 2; // #dr727-v0 - template static const inline int w = 1; // expected-error 0-1{{C++17 extension}} - template<> static const inline int w<0> = 2; // expected-error 0-1{{C++17 extension}} + template static const inline int w = 1; + // cxx14-error@-1 {{inline variables are a C++17 extension}} + template<> static const inline int w<0> = 2; + // cxx14-error@-1 {{inline variables are a C++17 extension}} }; template<> template constexpr int B<0>::u = 3; - template<> template<> constexpr int B<0>::u<0> = 4; // expected-error {{already has an initializer}} + template<> template<> constexpr int B<0>::u<0> = 4; + // since-cxx14-error@-1 {{static data member 'u' already has an initializer}} + // since-cxx14-note@#dr727-u0 {{previous initialization is here}} template<> template constexpr int B<0>::v = 3; template<> template<> constexpr int B<0>::v<0> = 4; -#if __cplusplus < 201702L - // expected-error@-2 {{already has an initializer}} - // expected-note@#v0 {{here}} -#endif + // cxx14-error@-1 {{static data member 'v' already has an initializer}} + // cxx14-note@#dr727-v0 {{previous initialization is here}} template<> template constexpr int B<0>::w = 3; template<> template<> constexpr int B<0>::w<0> = 4; @@ -182,10 +244,8 @@ namespace dr727 { // dr727: partial static_assert(B<1>().v<0> == 2, ""); static_assert(B<0>().v<1> == 3, ""); static_assert(B<0>().v<0> == 4, ""); -#if __cplusplus < 201702L - // expected-error@-2 {{failed}} \ - // expected-note@-2 {{evaluates to '2 == 4'}} -#endif + // cxx14-error@-1 {{static assertion failed due to requirement 'dr727::mixed_inner_outer_specialization::B<0>().v<0> == 4'}} + // cxx14-note@-2 {{expression evaluates to '2 == 4'}} static_assert(B<1>().w<1> == 1, ""); static_assert(B<1>().w<0> == 2, ""); @@ -205,13 +265,23 @@ namespace dr727 { // dr727: partial template<> int f2() {} template<> int f2() {} - template static int v1; // expected-error 0-1{{C++14 extension}} - template<> static int v1; // expected-note {{previous}} - template<> static int v1; // expected-error {{duplicate member}} - - template static inline int v2; // expected-error 0-1{{C++17 extension}} expected-error 0-1{{C++14 extension}} - template<> static inline int v2; // expected-error 0-1{{C++17 extension}} expected-note {{previous}} - template<> static inline int v2; // expected-error 0-1{{C++17 extension}} expected-error {{duplicate member}} + template static int v1; + // cxx98-11-error@-1 {{variable templates are a C++14 extension}} + template<> static int v1; // #dr727-v1-T + template<> static int v1; + // expected-error@-1 {{duplicate member 'v1'}} + // expected-note@#dr727-Collision-int-int {{in instantiation of template class 'dr727::Collision' requested here}} + // expected-note@#dr727-v1-T {{previous}} + + template static inline int v2; + // cxx98-11-error@-1 {{variable templates are a C++14 extension}} + // cxx98-14-error@-2 {{inline variables are a C++17 extension}} + template<> static inline int v2; // #dr727-v2-T + // cxx98-14-error@-1 {{inline variables are a C++17 extension}} + template<> static inline int v2; + // cxx98-14-error@-1 {{inline variables are a C++17 extension}} + // expected-error@-2 {{duplicate member 'v2'}} + // expected-note@#dr727-v2-T {{previous declaration is here}} // FIXME: Missing diagnostic for duplicate class explicit specialization. template struct S1; @@ -219,10 +289,12 @@ namespace dr727 { // dr727: partial template<> struct S1; template struct S2; - template<> struct S2 {}; // expected-note {{previous}} - template<> struct S2 {}; // expected-error {{redefinition}} + template<> struct S2 {}; // #dr727-S2-T + template<> struct S2 {}; + // expected-error@-1 {{redefinition of 'S2'}} + // expected-note@#dr727-S2-T {{previous}} }; - Collision c; // expected-note {{in instantiation of}} + Collision c; // #dr727-Collision-int-int } namespace dr777 { // dr777: 3.7 diff --git a/clang/test/CXX/drs/dr8xx.cpp b/clang/test/CXX/drs/dr8xx.cpp index d79430f9e03c..b031e65095cd 100644 --- a/clang/test/CXX/drs/dr8xx.cpp +++ b/clang/test/CXX/drs/dr8xx.cpp @@ -1,30 +1,27 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s -// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors 2>&1 | FileCheck %s +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +#if __cplusplus == 199711L // expected-no-diagnostics +#endif -namespace dr873 { // dr873: yes +namespace dr873 { // dr873: 3.0 #if __cplusplus >= 201103L template void f(T &&); -template <> void f(int &) {} // #1 -template <> void f(int &&) {} // #2 +template <> void f(int &) = delete; // #dr873-lvalue-ref +template <> void f(int &&) = delete; // #dr873-rvalue-ref void g(int i) { - f(i); // calls f(int&), i.e., #1 -#pragma clang __debug dump f(i) - // CHECK: CallExpr {{.*}} - // CHECK-NEXT: |-ImplicitCastExpr {{.*}} - // CHECK-NEXT: | `-DeclRefExpr {{.*}} 'f' 'void (int &)' {{.*}} - - f(0); // calls f(int&&), i.e., #2 -#pragma clang __debug dump f(0) - // CHECK: CallExpr {{.*}} - // CHECK-NEXT: |-ImplicitCastExpr {{.*}} - // CHECK-NEXT: | `-DeclRefExpr {{.*}} 'f' 'void (int &&)' {{.*}} + f(i); // calls f(int&) + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr873-lvalue-ref {{candidate function [with T = int &] has been implicitly deleted}} + f(0); // calls f(int&&) + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr873-rvalue-ref {{candidate function [with T = int] has been implicitly deleted}} } #endif } // namespace dr873 diff --git a/clang/test/CXX/drs/dr9xx.cpp b/clang/test/CXX/drs/dr9xx.cpp index fb13ef2967f8..7ea295a684b7 100644 --- a/clang/test/CXX/drs/dr9xx.cpp +++ b/clang/test/CXX/drs/dr9xx.cpp @@ -1,9 +1,9 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors namespace std { __extension__ typedef __SIZE_TYPE__ size_t; @@ -43,30 +43,34 @@ namespace dr948 { // dr948: 3.7 #endif } -namespace dr952 { // dr952: yes +namespace dr952 { // dr952: 2.8 namespace example1 { struct A { - typedef int I; // #dr952-typedef-decl + typedef int I; // #dr952-I }; -struct B : private A { // #dr952-inheritance +struct B : private A { // #dr952-B }; struct C : B { void f() { - I i1; // expected-error {{private member}} - // expected-note@#dr952-inheritance {{constrained by private inheritance}} - // expected-note@#dr952-typedef-decl {{declared here}} + I i1; + // expected-error@-1 {{'I' is a private member of 'dr952::example1::A'}} + // expected-note@#dr952-B {{constrained by private inheritance here}} + // expected-note@#dr952-I {{member is declared here}} } - I i2; // expected-error {{private member}} - // expected-note@#dr952-inheritance {{constrained by private inheritance}} - // expected-note@#dr952-typedef-decl {{declared here}} + I i2; + // expected-error@-1 {{'I' is a private member of 'dr952::example1::A'}} + // expected-note@#dr952-B {{constrained by private inheritance here}} + // expected-note@#dr952-I {{member is declared here}} struct D { - I i3; // expected-error {{private member}} - // expected-note@#dr952-inheritance {{constrained by private inheritance}} - // expected-note@#dr952-typedef-decl {{declared here}} + I i3; + // expected-error@-1 {{'I' is a private member of 'dr952::example1::A'}} + // expected-note@#dr952-B {{constrained by private inheritance here}} + // expected-note@#dr952-I {{member is declared here}} void g() { - I i4; // expected-error {{private member}} - // expected-note@#dr952-inheritance {{constrained by private inheritance}} - // expected-note@#dr952-typedef-decl {{declared here}} + I i4; + // expected-error@-1 {{'I' is a private member of 'dr952::example1::A'}} + // expected-note@#dr952-B {{constrained by private inheritance here}} + // expected-note@#dr952-I {{member is declared here}} } }; }; @@ -91,10 +95,10 @@ namespace dr974 { // dr974: yes } namespace dr977 { // dr977: yes -enum E { e = E() }; +enum E { e = E() }; // #dr977-E #if !defined(_WIN32) || defined(__MINGW32__) -// expected-error@-2 {{invalid use of incomplete type 'E'}} -// expected-note@-3 {{definition of 'dr977::E' is not complete until the closing '}'}} +// expected-error@#dr977-E {{invalid use of incomplete type 'E'}} +// expected-note@#dr977-E {{definition of 'dr977::E' is not complete until the closing '}'}} #endif #if __cplusplus >= 201103L enum E2 : int { e2 = E2() }; @@ -105,23 +109,29 @@ enum struct E4 : int { e = static_cast(E4()) }; namespace dr990 { // dr990: 3.5 #if __cplusplus >= 201103L - struct A { // expected-note 2{{candidate}} - A(std::initializer_list); // expected-note {{candidate}} + struct A { // #dr990-A + A(std::initializer_list); // #dr990-A-init-list }; struct B { A a; }; B b1 { }; - B b2 { 1 }; // expected-error {{no viable conversion from 'int' to 'A'}} + B b2 { 1 }; + // since-cxx11-error@-1 {{no viable conversion from 'int' to 'A'}} + // since-cxx11-note@#dr990-A {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int' to 'const A &' for 1st argument}} + // since-cxx11-note@#dr990-A {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'int' to 'A &&' for 1st argument}} + // since-cxx11-note@#dr990-A-init-list {{candidate constructor not viable: no known conversion from 'int' to 'std::initializer_list' for 1st argument}} B b3 { { 1 } }; struct C { C(); C(int); - C(std::initializer_list) = delete; // expected-note {{here}} + C(std::initializer_list) = delete; // #dr990-deleted }; C c1[3] { 1 }; // ok - C c2[3] { 1, {2} }; // expected-error {{call to deleted}} + C c2[3] { 1, {2} }; + // since-cxx11-error@-1 {{call to deleted constructor of 'C'}} + // since-cxx11-note@#dr990-deleted {{'C' has been explicitly marked deleted here}} struct D { D(); diff --git a/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp b/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp index ab1b9f7a73ee..1cf9e1c9f9c0 100644 --- a/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp +++ b/clang/test/CXX/temp/temp.decls/temp.friend/p1.cpp @@ -17,7 +17,7 @@ public: for (U count = n.count_; count; --count) x += a; return x; - } + } }; friend Num operator+(const Num &a, const Num &b) { @@ -145,7 +145,7 @@ namespace test5 { namespace Dependent { template class X; - template + template X operator+(const X&, const T*); template class X { @@ -249,7 +249,7 @@ namespace test11 { }; template struct Foo::IteratorImpl; - template struct Foo::IteratorImpl; + template struct Foo::IteratorImpl; } // PR6827 diff --git a/clang/test/CodeGen/RISCV/riscv-func-attr-target.c b/clang/test/CodeGen/RISCV/riscv-func-attr-target.c index 74bc5f2ac704..506acaba6874 100644 --- a/clang/test/CodeGen/RISCV/riscv-func-attr-target.c +++ b/clang/test/CodeGen/RISCV/riscv-func-attr-target.c @@ -1,6 +1,7 @@ // REQUIRES: riscv-registered-target // RUN: %clang_cc1 -triple riscv64 -target-feature +zifencei -target-feature +m \ -// RUN: -target-feature +a -target-feature +save-restore \ +// RUN: -target-feature +a -target-feature +save-restore -target-feature -zbb \ +// RUN: -target-feature -relax -target-feature -zfa \ // RUN: -emit-llvm %s -o - | FileCheck %s // CHECK-LABEL: define dso_local void @testDefault @@ -35,12 +36,12 @@ testAttrFullArchAndAttrCpu() {} __attribute__((target("cpu=sifive-u54"))) void testAttrCpuOnly() {} //. -// CHECK: attributes #0 = { {{.*}}"target-features"="+64bit,+a,+m,+save-restore,+zifencei" } -// CHECK: attributes #1 = { {{.*}}"target-cpu"="rocket-rv64" "target-features"="+64bit,+a,+d,+f,+m,+save-restore,+v,+zicsr,+zifencei,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b" "tune-cpu"="generic-rv64" } -// CHECK: attributes #2 = { {{.*}}"target-features"="+64bit,+a,+m,+save-restore,+zbb,+zifencei" } -// CHECK: attributes #3 = { {{.*}}"target-features"="+64bit,+a,+d,+experimental-zicond,+f,+m,+save-restore,+v,+zbb,+zicsr,+zifencei,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b" } -// CHECK: attributes #4 = { {{.*}}"target-features"="+64bit,+a,+c,+d,+f,+m,+save-restore,+zbb,+zicsr,+zifencei" } -// CHECK: attributes #5 = { {{.*}}"target-features"="+64bit,+m,+save-restore" } -// CHECK: attributes #6 = { {{.*}}"target-cpu"="sifive-u54" "target-features"="+64bit,+a,+m,+save-restore,+zbb,+zifencei" } -// CHECK: attributes #7 = { {{.*}}"target-cpu"="sifive-u54" "target-features"="+64bit,+m,+save-restore" } -// CHECK: attributes #8 = { {{.*}}"target-cpu"="sifive-u54" "target-features"="+64bit,+a,+c,+d,+f,+m,+save-restore,+zicsr,+zifencei" } +// CHECK: attributes #0 = { {{.*}}"target-features"="+64bit,+a,+m,+save-restore,+zifencei,-relax,-zbb,-zfa" } +// CHECK: attributes #1 = { {{.*}}"target-cpu"="rocket-rv64" "target-features"="+64bit,+a,+d,+f,+m,+save-restore,+v,+zicsr,+zifencei,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b,-relax,-zbb,-zfa" "tune-cpu"="generic-rv64" } +// CHECK: attributes #2 = { {{.*}}"target-features"="+64bit,+a,+m,+save-restore,+zbb,+zifencei,-relax,-zfa" } +// CHECK: attributes #3 = { {{.*}}"target-features"="+64bit,+a,+d,+experimental-zicond,+f,+m,+save-restore,+v,+zbb,+zicsr,+zifencei,+zve32f,+zve32x,+zve64d,+zve64f,+zve64x,+zvl128b,+zvl32b,+zvl64b,-relax,-zfa" } +// CHECK: attributes #4 = { {{.*}}"target-features"="+64bit,+a,+c,+d,+f,+m,+save-restore,+zbb,+zicsr,+zifencei,-relax,-zfa" } +// CHECK: attributes #5 = { {{.*}}"target-features"="+64bit,+m,+save-restore,-relax,-zbb,-zfa" } +// CHECK: attributes #6 = { {{.*}}"target-cpu"="sifive-u54" "target-features"="+64bit,+a,+m,+save-restore,+zbb,+zifencei,-relax,-zfa" } +// CHECK: attributes #7 = { {{.*}}"target-cpu"="sifive-u54" "target-features"="+64bit,+m,+save-restore,-relax,-zbb,-zfa" } +// CHECK: attributes #8 = { {{.*}}"target-cpu"="sifive-u54" "target-features"="+64bit,+a,+c,+d,+f,+m,+save-restore,+zicsr,+zifencei,-relax,-zbb,-zfa" } diff --git a/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/rvv-error.c b/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/rvv-error.c index 1a29acbf3ba9..6ec9b0579976 100644 --- a/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/rvv-error.c +++ b/clang/test/CodeGen/RISCV/rvv-intrinsics-handcrafted/rvv-error.c @@ -11,7 +11,7 @@ // CHECK-RV64V-NEXT: ret i32 [[CONV]] // -// CHECK-RV64-ERR: error: builtin requires at least one of the following extensions to be enabled: 'Zve32x' +// CHECK-RV64-ERR: error: builtin requires at least one of the following extensions: 'Zve32x' int test() { return __builtin_rvv_vsetvli(1, 0, 0); diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c new file mode 100644 index 000000000000..82b30e8bbe9b --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_reinterpret_svcount_svbool.c @@ -0,0 +1,47 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py + +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -p mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin.§ +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +// CHECK-LABEL: @test_svreinterpret_svbool_svcnt( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[CNT:%.*]]) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z31test_svreinterpret_svbool_svcntu11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.taarch64.svcountt(target("aarch64.svcount") [[CNT:%.*]]) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbool_t test_svreinterpret_svbool_svcnt(svcount_t cnt) __arm_streaming_compatible +{ + return SVE_ACLE_FUNC(svreinterpret,_b,,)(cnt); +} + +// CHECK-LABEL: @test_svreinterpret_svcnt_svbool( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[PG:%.*]]) +// CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z31test_svreinterpret_svcnt_svboolu10__SVBool_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[PG:%.*]]) +// CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP0]] +// +svcount_t test_svreinterpret_svcnt_svbool(svbool_t pg) __arm_streaming_compatible +{ + return SVE_ACLE_FUNC(svreinterpret,_c,,)(pg); +} diff --git a/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_qrshr.c b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_qrshr.c new file mode 100644 index 000000000000..50cac4888789 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sme2-intrinsics/acle_sme2_vector_qrshr.c @@ -0,0 +1,343 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// REQUIRES: aarch64-registered-target + +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s | opt -S -passes=mem2reg,instcombine,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s | opt -S -passes=mem2reg,instcombine,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve -target-feature +sme2 -target-feature +sme-f64f64 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED,A5) A1##A3##A5 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4,A5) A1##A2##A3##A4##A5 +#endif + +// SVQRSHR + +// CHECK-LABEL: @test_svsqrshr_u16_u32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.uqrshr.x2.nxv4i32( [[TMP0]], [[TMP1]], i32 16) +// CHECK-NEXT: ret [[TMP2]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshr_u16_u32_x412svuint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.uqrshr.x2.nxv4i32( [[TMP0]], [[TMP1]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP2]] +// +svuint16_t test_svsqrshr_u16_u32_x4(svuint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshr,_n,_u16,_u32_x2,)(zn, 16); +} + +// CHECK-LABEL: @test_svsqrshr_s16_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.sqrshr.x2.nxv4i32( [[TMP0]], [[TMP1]], i32 16) +// CHECK-NEXT: ret [[TMP2]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshr_s16_s32_x411svint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.sqrshr.x2.nxv4i32( [[TMP0]], [[TMP1]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP2]] +// +svint16_t test_svsqrshr_s16_s32_x4(svint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshr,_n,_s16,_s32_x2,)(zn, 16); +} + +// CHECK-LABEL: @test_svsqrshr_u8_u32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshr.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z23test_svsqrshr_u8_u32_x412svuint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshr.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint8_t test_svsqrshr_u8_u32_x4(svuint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshr,_n,_u8,_u32_x4,)(zn, 8); +} + +// CHECK-LABEL: @test_svsqrshr_s8_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshr.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z23test_svsqrshr_s8_s32_x411svint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshr.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svint8_t test_svsqrshr_s8_s32_x4(svint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshr,_n,_s8,_s32_x4,)(zn, 8); +} + +// CHECK-LABEL: @test_svsqrshr_u16_u64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshr.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshr_u16_u64_x412svuint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshr.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint16_t test_svsqrshr_u16_u64_x4(svuint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshr,_n,_u16,_u64_x4,)(zn, 16); +} + +// CHECK-LABEL: @test_svsqrshr_s16_s64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshr.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshr_s16_s64_x411svint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshr.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svint16_t test_svsqrshr_s16_s64_x4(svint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshr,_n,_s16,_s64_x4,)(zn, 16); +} + +// SVQRSHRN + +// CHECK-LABEL: @test_svsqrshrn_u8_u32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshrn.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshrn_u8_u32_x412svuint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshrn.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint8_t test_svsqrshrn_u8_u32_x4(svuint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshrn,_n,_u8,_u32_x4,)(zn, 8); +} + +// CHECK-LABEL: @test_svsqrshrn_s8_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrn.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshrn_s8_s32_x411svint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrn.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svint8_t test_svsqrshrn_s8_s32_x4(svint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshrn,_n,_s8,_s32_x4,)(zn, 8); +} + +// CHECK-LABEL: @test_svsqrshrn_u16_u64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshrn.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z25test_svsqrshrn_u16_u64_x412svuint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.uqrshrn.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint16_t test_svsqrshrn_u16_u64_x4(svuint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshrn,_n,_u16,_u64_x4,)(zn, 16); +} + +// CHECK-LABEL: @test_svsqrshrn_s16_s64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrn.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z25test_svsqrshrn_s16_s64_x411svint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrn.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svint16_t test_svsqrshrn_s16_s64_x4(svint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshrn,_n,_s16,_s64_x4,)(zn, 16); +} + +// SVSQRSHRU + +// CHECK-LABEL: @test_svsvqrshru_u16_s32_x2( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.sqrshru.x2.nxv4i32( [[TMP0]], [[TMP1]], i32 16) +// CHECK-NEXT: ret [[TMP2]] +// +// CPP-CHECK-LABEL: @_Z26test_svsvqrshru_u16_s32_x211svint32x2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv8i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.sqrshru.x2.nxv4i32( [[TMP0]], [[TMP1]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP2]] +// +svuint16_t test_svsvqrshru_u16_s32_x2(svint32x2_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshru,_n,_u16,_s32_x2,)(zn, 16); +} + +// CHECK-LABEL: @test_svsqrshru_u8_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshru.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z24test_svsqrshru_u8_s32_x411svint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshru.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 8) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint8_t test_svsqrshru_u8_s32_x4(svint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshru,_n,_u8,_s32_x4,)(zn, 8); +} + +// CHECK-LABEL: @test_svsqrshru_u16_s64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshru.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z25test_svsqrshru_u16_s64_x411svint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshru.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint16_t test_svsqrshru_u16_s64_x4(svint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshru,_n,_u16,_s64_x4,)(zn, 16); +} + +// SQRSHRUN x 4 + +// CHECK-LABEL: @test_svsqrshrun_u8_s32_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrun.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 32) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z25test_svsqrshrun_u8_s32_x411svint32x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 8) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv4i32.nxv16i32( [[ZN]], i64 12) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrun.x4.nxv4i32( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 32) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint8_t test_svsqrshrun_u8_s32_x4(svint32x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshrun,_n,_u8,_s32_x4,)(zn, 32); +} + +// CHECK-LABEL: @test_svsqrshrun_u16_s64_x4( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrun.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 64) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z26test_svsqrshrun_u16_s64_x411svint64x4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 2) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 4) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.extract.nxv2i64.nxv8i64( [[ZN]], i64 6) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.aarch64.sve.sqrshrun.x4.nxv2i64( [[TMP0]], [[TMP1]], [[TMP2]], [[TMP3]], i32 64) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svuint16_t test_svsqrshrun_u16_s64_x4(svint64x4_t zn) __arm_streaming { + return SVE_ACLE_FUNC(svqrshrun,_n,_u16,_s64_x4,)(zn, 64); +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create2_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create2_bool.c new file mode 100644 index 000000000000..eb5a19b1d9d3 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create2_bool.c @@ -0,0 +1,38 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +// REQUIRES: aarch64-registered-target + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +// CHECK-LABEL: @test_svcreate2_s8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[X0:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP0]], [[X1:%.*]], i64 16) +// CHECK-NEXT: ret [[TMP1]] +// +// CPP-CHECK-LABEL: @_Z17test_svcreate2_s8u10__SVBool_tS_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[X0:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP0]], [[X1:%.*]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP1]] +// +svboolx2_t test_svcreate2_s8(svbool_t x0, svbool_t x1) +{ + return SVE_ACLE_FUNC(svcreate2,_b8,,)(x0, x1); +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create4_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create4_bool.c new file mode 100644 index 000000000000..3f99ee6e79c0 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_create4_bool.c @@ -0,0 +1,42 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +// REQUIRES: aarch64-registered-target + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +// CHECK-LABEL: @test_svcreate4_b8( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( poison, [[X0:%.*]], i64 0) +// CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TMP0]], [[X1:%.*]], i64 16) +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TMP1]], [[X2:%.*]], i64 32) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TMP2]], [[X4:%.*]], i64 48) +// CHECK-NEXT: ret [[TMP3]] +// +// CPP-CHECK-LABEL: @_Z17test_svcreate4_b8u10__SVBool_tS_S_S_( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( poison, [[X0:%.*]], i64 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TMP0]], [[X1:%.*]], i64 16) +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TMP1]], [[X2:%.*]], i64 32) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TMP2]], [[X4:%.*]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP3]] +// +svboolx4_t test_svcreate4_b8(svbool_t x0, svbool_t x1, svbool_t x2, svbool_t x4) +{ + return SVE_ACLE_FUNC(svcreate4,_b8,,)(x0, x1, x2, x4); +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get2_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get2_bool.c new file mode 100644 index 000000000000..5d38f72b34b1 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get2_bool.c @@ -0,0 +1,49 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// REQUIRES: aarch64-registered-target +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +// CHECK-LABEL: @test_svget2_b8_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv32i1( [[TUPLE:%.*]], i64 0) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svget2_b8_010svboolx2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv32i1( [[TUPLE:%.*]], i64 0) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbool_t test_svget2_b8_0(svboolx2_t tuple) +{ + return SVE_ACLE_FUNC(svget2,_b8,,)(tuple, 0); +} + +// CHECK-LABEL: @test_svget2_b8_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv32i1( [[TUPLE:%.*]], i64 16) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svget2_b8_110svboolx2_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv32i1( [[TUPLE:%.*]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbool_t test_svget2_b8_1(svboolx2_t tuple) +{ + return SVE_ACLE_FUNC(svget2,_b8,,)(tuple, 1); +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get4_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get4_bool.c new file mode 100644 index 000000000000..821a5be3254e --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_get4_bool.c @@ -0,0 +1,72 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +// REQUIRES: aarch64-registered-target + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +// NOTE: For these tests clang converts the struct parameter into +// several parameters, one for each member of the original struct. +// CHECK-LABEL: @test_svget4_b8_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv64i1( [[TUPLE:%.*]], i64 0) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svget4_b8_010svboolx4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv64i1( [[TUPLE:%.*]], i64 0) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbool_t test_svget4_b8_0(svboolx4_t tuple) +{ + return SVE_ACLE_FUNC(svget4,_b8,,)(tuple, 0); +} + +// NOTE: For these tests clang converts the struct parameter into +// several parameters, one for each member of the original struct. +// CHECK-LABEL: @test_svget4_b8_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv64i1( [[TUPLE:%.*]], i64 16) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svget4_b8_110svboolx4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv64i1( [[TUPLE:%.*]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbool_t test_svget4_b8_1(svboolx4_t tuple) +{ + return SVE_ACLE_FUNC(svget4,_b8,,)(tuple, 1); +} + +// NOTE: For these tests clang converts the struct parameter into +// several parameters, one for each member of the original struct. +// CHECK-LABEL: @test_svget4_b8_3( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv64i1( [[TUPLE:%.*]], i64 48) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svget4_b8_310svboolx4_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.extract.nxv16i1.nxv64i1( [[TUPLE:%.*]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svbool_t test_svget4_b8_3(svboolx4_t tuple) +{ + return SVE_ACLE_FUNC(svget4,_b8,,)(tuple, 3); +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c index fe15d5a9db81..a3206029019c 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_pext.c @@ -1,10 +1,17 @@ // NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py // REQUIRES: aarch64-registered-target +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -DTEST_SME2 -target-feature +sve -target-feature +sme2 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s // RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s -// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include +#ifndef TEST_SME2 +#define ATTR +#else +#define ATTR __arm_streaming +#endif + // CHECK-LABEL: @test_svpext_lane_c8_0( // CHECK-NEXT: entry: // CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 0) @@ -15,7 +22,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 0) // CPP-CHECK-NEXT: ret [[TMP0]] // -svbool_t test_svpext_lane_c8_0(svcount_t c) { +svbool_t test_svpext_lane_c8_0(svcount_t c) ATTR { return svpext_lane_c8(c, 0); } @@ -29,7 +36,7 @@ svbool_t test_svpext_lane_c8_0(svcount_t c) { // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.pext.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 3) // CPP-CHECK-NEXT: ret [[TMP0]] // -svbool_t test_svpext_lane_c8_3(svcount_t c) { +svbool_t test_svpext_lane_c8_3(svcount_t c) ATTR { return svpext_lane_c8(c, 3); } @@ -45,7 +52,7 @@ svbool_t test_svpext_lane_c8_3(svcount_t c) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP0]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpext_lane_c16_0(svcount_t c) { +svbool_t test_svpext_lane_c16_0(svcount_t c) ATTR { return svpext_lane_c16(c, 0); } @@ -61,7 +68,7 @@ svbool_t test_svpext_lane_c16_0(svcount_t c) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP0]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpext_lane_c16_3(svcount_t c) { +svbool_t test_svpext_lane_c16_3(svcount_t c) ATTR { return svpext_lane_c16(c, 3); } @@ -77,7 +84,7 @@ svbool_t test_svpext_lane_c16_3(svcount_t c) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP0]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpext_lane_c32_0(svcount_t c) { +svbool_t test_svpext_lane_c32_0(svcount_t c) ATTR { return svpext_lane_c32(c, 0); } @@ -93,7 +100,7 @@ svbool_t test_svpext_lane_c32_0(svcount_t c) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP0]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpext_lane_c32_3(svcount_t c) { +svbool_t test_svpext_lane_c32_3(svcount_t c) ATTR { return svpext_lane_c32(c, 3); } @@ -109,7 +116,7 @@ svbool_t test_svpext_lane_c32_3(svcount_t c) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP0]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpext_lane_c64_0(svcount_t c) { +svbool_t test_svpext_lane_c64_0(svcount_t c) ATTR { return svpext_lane_c64(c, 0); } @@ -125,7 +132,7 @@ svbool_t test_svpext_lane_c64_0(svcount_t c) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP0]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpext_lane_c64_3(svcount_t c) { +svbool_t test_svpext_lane_c64_3(svcount_t c) ATTR { return svpext_lane_c64(c, 3); } @@ -147,6 +154,184 @@ svbool_t test_svpext_lane_c64_3(svcount_t c) { // CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) // CPP-CHECK-NEXT: ret [[TMP4]] // -svboolx2_t test_svpext_lane_c8_x2_0(svcount_t c) { +svboolx2_t test_svpext_lane_c8_x2_0(svcount_t c) ATTR { return svpext_lane_c8_x2(c, 0); } + +// CHECK-LABEL: @test_svpext_lane_c8_x2_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CHECK-NEXT: ret [[TMP4]] +// +// CPP-CHECK-LABEL: @_Z24test_svpext_lane_c8_x2_1u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv16i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP1]], i64 0) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP4:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP2]], [[TMP3]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP4]] +// +svboolx2_t test_svpext_lane_c8_x2_1(svcount_t c) ATTR { + return svpext_lane_c8_x2(c, 1); +} + +// CHECK-LABEL: @test_svpext_lane_c16_x2_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv8i1(target("aarch64.svcount") [[C:%.*]], i32 0) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z25test_svpext_lane_c16_x2_0u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv8i1(target("aarch64.svcount") [[C:%.*]], i32 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svpext_lane_c16_x2_0(svcount_t c) ATTR { + return svpext_lane_c16_x2(c, 0); +} + +// CHECK-LABEL: @test_svpext_lane_c16_x2_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv8i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z25test_svpext_lane_c16_x2_1u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv8i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv8i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svpext_lane_c16_x2_1(svcount_t c) ATTR { + return svpext_lane_c16_x2(c, 1); +} + +// CHECK-LABEL: @test_svpext_lane_c32_x2_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv4i1(target("aarch64.svcount") [[C:%.*]], i32 0) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z25test_svpext_lane_c32_x2_0u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv4i1(target("aarch64.svcount") [[C:%.*]], i32 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svpext_lane_c32_x2_0(svcount_t c) ATTR { + return svpext_lane_c32_x2(c, 0); +} + +// CHECK-LABEL: @test_svpext_lane_c32_x2_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv4i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z25test_svpext_lane_c32_x2_1u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv4i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv4i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svpext_lane_c32_x2_1(svcount_t c) ATTR { + return svpext_lane_c32_x2(c, 1); +} + +// CHECK-LABEL: @test_svpext_lane_c64_x2_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv2i1(target("aarch64.svcount") [[C:%.*]], i32 0) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z25test_svpext_lane_c64_x2_0u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv2i1(target("aarch64.svcount") [[C:%.*]], i32 0) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svpext_lane_c64_x2_0(svcount_t c) ATTR { + return svpext_lane_c64_x2(c, 0); +} + +// CHECK-LABEL: @test_svpext_lane_c64_x2_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv2i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CHECK-NEXT: ret [[TMP6]] +// +// CPP-CHECK-LABEL: @_Z25test_svpext_lane_c64_x2_1u11__SVCount_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call { , } @llvm.aarch64.sve.pext.x2.nxv2i1(target("aarch64.svcount") [[C:%.*]], i32 1) +// CPP-CHECK-NEXT: [[TMP1:%.*]] = extractvalue { , } [[TMP0]], 0 +// CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP1]]) +// CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( poison, [[TMP2]], i64 0) +// CPP-CHECK-NEXT: [[TMP4:%.*]] = extractvalue { , } [[TMP0]], 1 +// CPP-CHECK-NEXT: [[TMP5:%.*]] = tail call @llvm.aarch64.sve.convert.to.svbool.nxv2i1( [[TMP4]]) +// CPP-CHECK-NEXT: [[TMP6:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TMP3]], [[TMP5]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP6]] +// +svboolx2_t test_svpext_lane_c64_x2_1(svcount_t c) ATTR { + return svpext_lane_c64_x2(c, 1); +} diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c index aa2a35c2fd25..73b7b0347dd9 100644 --- a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_psel.c @@ -5,6 +5,11 @@ // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ // RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK // RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ +// RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - %s | FileCheck %s +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu \ +// RUN: -target-feature +sve2p1 -S -O1 -Werror -emit-llvm -o - -x c++ %s | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -fclang-abi-compat=latest -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s #include @@ -20,7 +25,7 @@ // CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.aarch64.sve.psel.nxv16i1( [[P1:%.*]], [[P2:%.*]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP0]] // -svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) { +svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_b8(p1, p2, idx + 15); } @@ -38,7 +43,7 @@ svbool_t test_svpsel_lane_b8(svbool_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv8i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) { +svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_b16(p1, p2, idx + 7); } @@ -56,7 +61,7 @@ svbool_t test_svpsel_lane_b16(svbool_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv4i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) { +svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_b32(p1, p2, idx + 3); } @@ -74,7 +79,7 @@ svbool_t test_svpsel_lane_b32(svbool_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP1:%.*]] = tail call @llvm.aarch64.sve.psel.nxv2i1( [[P1:%.*]], [[TMP0]], i32 [[ADD]]) // CPP-CHECK-NEXT: ret [[TMP1]] // -svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) { +svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_b64(p1, p2, idx + 1); } @@ -94,7 +99,7 @@ svbool_t test_svpsel_lane_b64(svbool_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP2:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP1]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP2]] // -svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) { +svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_c8(p1, p2, idx + 15); } @@ -116,7 +121,7 @@ svcount_t test_svpsel_lane_c8(svcount_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) { +svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_c16(p1, p2, idx + 7); } @@ -138,7 +143,7 @@ svcount_t test_svpsel_lane_c16(svcount_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) { +svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_c32(p1, p2, idx + 3); } @@ -160,6 +165,6 @@ svcount_t test_svpsel_lane_c32(svcount_t p1, svbool_t p2, uint32_t idx) { // CPP-CHECK-NEXT: [[TMP3:%.*]] = tail call target("aarch64.svcount") @llvm.aarch64.sve.convert.from.svbool.taarch64.svcountt( [[TMP2]]) // CPP-CHECK-NEXT: ret target("aarch64.svcount") [[TMP3]] // -svcount_t test_svpsel_lane_c64(svcount_t p1, svbool_t p2, uint32_t idx) { +svcount_t test_svpsel_lane_c64(svcount_t p1, svbool_t p2, uint32_t idx) __arm_streaming_compatible { return svpsel_lane_c64(p1, p2, idx + 1); } diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set2_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set2_bool.c new file mode 100644 index 000000000000..298e5b80ceb0 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set2_bool.c @@ -0,0 +1,52 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s + +// REQUIRES: aarch64-registered-target + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + +// CHECK-LABEL: @test_svset2_b8_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 0) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svset2_b8_010svboolx2_tu10__SVBool_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 0) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svboolx2_t test_svset2_b8_0(svboolx2_t tuple, svbool_t x) +{ + return SVE_ACLE_FUNC(svset2,_b8,,)(tuple, 0, x); +} + +// CHECK-LABEL: @test_svset2_b8_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 16) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svset2_b8_110svboolx2_tu10__SVBool_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv32i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svboolx2_t test_svset2_b8_1(svboolx2_t tuple, svbool_t x) +{ + return SVE_ACLE_FUNC(svset2,_b8,,)(tuple, 1, x); +} + diff --git a/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set4_bool.c b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set4_bool.c new file mode 100644 index 000000000000..746a8280ac53 --- /dev/null +++ b/clang/test/CodeGen/aarch64-sve2p1-intrinsics/acle_sve2p1_set4_bool.c @@ -0,0 +1,66 @@ +// NOTE: Assertions have been autogenerated by utils/update_cc_test_checks.py +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - %s \ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s +// RUN: %clang_cc1 -DSVE_OVERLOADED_FORMS -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -emit-llvm -o - -x c++ %s\ +// RUN: | opt -S -passes=mem2reg,tailcallelim | FileCheck %s -check-prefix=CPP-CHECK +// RUN: %clang_cc1 -triple aarch64-none-linux-gnu -target-feature +sve2p1 -S -disable-O0-optnone -Werror -Wall -o /dev/null %s +// REQUIRES: aarch64-registered-target + +#include + +#ifdef SVE_OVERLOADED_FORMS +// A simple used,unused... macro, long enough to represent any SVE builtin. +#define SVE_ACLE_FUNC(A1,A2_UNUSED,A3,A4_UNUSED) A1##A3 +#else +#define SVE_ACLE_FUNC(A1,A2,A3,A4) A1##A2##A3##A4 +#endif + + +// CHECK-LABEL: @test_svset4_b8_0( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 0) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svset4_b8_010svboolx4_tu10__SVBool_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 0) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svboolx4_t test_svset4_b8_0(svboolx4_t tuple, svbool_t x) +{ + return SVE_ACLE_FUNC(svset4,_b8,,)(tuple, 0, x); +} + +// CHECK-LABEL: @test_svset4_b8_1( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 16) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svset4_b8_110svboolx4_tu10__SVBool_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 16) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svboolx4_t test_svset4_b8_1(svboolx4_t tuple, svbool_t x) +{ + return SVE_ACLE_FUNC(svset4,_b8,,)(tuple, 1, x); +} + +// CHECK-LABEL: @test_svset4_b8_3( +// CHECK-NEXT: entry: +// CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 48) +// CHECK-NEXT: ret [[TMP0]] +// +// CPP-CHECK-LABEL: @_Z16test_svset4_b8_310svboolx4_tu10__SVBool_t( +// CPP-CHECK-NEXT: entry: +// CPP-CHECK-NEXT: [[TMP0:%.*]] = tail call @llvm.vector.insert.nxv64i1.nxv16i1( [[TUPLE:%.*]], [[X:%.*]], i64 48) +// CPP-CHECK-NEXT: ret [[TMP0]] +// +svboolx4_t test_svset4_b8_3(svboolx4_t tuple, svbool_t x) +{ + return SVE_ACLE_FUNC(svset4,_b8,,)(tuple, 3, x); +} diff --git a/clang/test/CodeGen/arm-target-features.c b/clang/test/CodeGen/arm-target-features.c index bd051059fb0c..ad4bfd45c408 100644 --- a/clang/test/CodeGen/arm-target-features.c +++ b/clang/test/CodeGen/arm-target-features.c @@ -113,6 +113,9 @@ // RUN: %clang_cc1 -triple thumb-linux-gnueabi -target-cpu cortex-m85 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-ARMV81M-CORTEX-M85-LINUX // CHECK-ARMV81M-CORTEX-M85-LINUX: "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+pacbti,+ras,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp" +// RUN: %clang_cc1 -triple thumb-linux-gnueabi -target-cpu cortex-m52 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-ARMV81M-CORTEX-M52-LINUX +// CHECK-ARMV81M-CORTEX-M52-LINUX: "target-features"="+armv8.1-m.main,+dsp,+fp-armv8d16,+fp-armv8d16sp,+fp16,+fp64,+fullfp16,+hwdiv,+lob,+mve,+mve.fp,+pacbti,+ras,+thumb-mode,+vfp2,+vfp2sp,+vfp3d16,+vfp3d16sp,+vfp4d16,+vfp4d16sp" + // RUN: %clang_cc1 -triple thumbv9.3a-linux-gnueabihf -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-ARCH93 // CHECK-ARCH93: "target-features"="+armv9.3-a,+thumb-mode,+v8.1a,+v8.2a,+v8.3a,+v8.4a,+v8.5a,+v8.6a,+v8.7a,+v8.8a,+v9.1a,+v9.2a,+v9.3a,+v9a" diff --git a/clang/test/CodeGen/complex-math.c b/clang/test/CodeGen/complex-math.c index c59baaa45236..a44aa0014a65 100644 --- a/clang/test/CodeGen/complex-math.c +++ b/clang/test/CodeGen/complex-math.c @@ -5,7 +5,7 @@ // RUN: %clang_cc1 %s -O0 -emit-llvm -triple armv7-none-linux-gnueabi -o - | FileCheck %s --check-prefix=ARM // RUN: %clang_cc1 %s -O0 -emit-llvm -triple armv7-none-linux-gnueabihf -o - | FileCheck %s --check-prefix=ARMHF // RUN: %clang_cc1 %s -O0 -emit-llvm -triple thumbv7k-apple-watchos2.0 -o - -target-abi aapcs16 | FileCheck %s --check-prefix=ARM7K -// RUN: %clang_cc1 %s -O0 -emit-llvm -triple aarch64-unknown-unknown -ffast-math -ffp-contract=fast -o - | FileCheck %s --check-prefix=AARCH64-FASTMATH +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple aarch64-unknown-unknown -ffast-math -ffp-contract=fast -complex-range=fortran -o - | FileCheck %s --check-prefix=AARCH64-FASTMATH // RUN: %clang_cc1 %s -O0 -emit-llvm -triple spir -o - | FileCheck %s --check-prefix=SPIR float _Complex add_float_rr(float a, float b) { @@ -135,24 +135,68 @@ float _Complex div_float_rc(float a, float _Complex b) { // SPIR: call spir_func {{.*}} @__divsc3( - // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD)) + // a / b = (A+iB) / (C+iD) = (E+iF) + // if (|C| >= |D|) + // DdC = D/C + // CpRD = C+DdC*D + // E = (A+B*DdC)/CpRD + // F = (B-A*DdC)/CpRD + // else + // CdD = C/D + // DpRC= D+CdD*C + // E = (A*CdD+B)/DpRC + // F = (B*CdD-A)/DpRC // AARCH64-FASTMATH-LABEL: @div_float_rc(float noundef nofpclass(nan inf) %a, [2 x float] noundef nofpclass(nan inf) alignstack(8) %b.coerce) - // A = a - // B = 0 - // - // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast float - // BD = 0 - // ACpBD = AC - // - // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast float - // - // BC = 0 - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast float - // - // AARCH64-FASTMATH: fdiv fast float - // AARCH64-FASTMATH: fdiv fast float + // |C| + // AARCH64-FASTMATH: call {{.*}}float @llvm.fabs.f32(float {{.*}}) + // |D| + // AARCH64-FASTMATH-NEXT: call {{.*}}float @llvm.fabs.f32(float {{.*}}) + // AARCH64-FASTMATH-NEXT: fcmp {{.*}}ugt float + // AARCH64-FASTMATH-NEXT: br i1 {{.*}}, label + // AARCH64-FASTMATH: abs_rhsr_greater_or_equal_abs_rhsi: + + // |C| >= |D| + // DdC=D/C + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // CpRD=C+CdC*D + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + + // A+BR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // B-AR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fsub {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: abs_rhsr_less_than_abs_rhsi: + + // |C| < |D| + // CdD=C/D + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // DpRC=D+CdD*C + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + + // (A*CdD+B)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // (BCdD-A)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fsub {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: complex_div: + // AARCH64-FASTMATH-NEXT: phi {{.*}}float + // AARCH64-FASTMATH-NEXT: phi {{.*}}float // AARCH64-FASTMATH: ret return a / b; } @@ -164,24 +208,68 @@ float _Complex div_float_cc(float _Complex a, float _Complex b) { // SPIR: call spir_func {{.*}} @__divsc3( - // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD)) + // a / b = (A+iB) / (C+iD) = (E+iF) + // if (|C| >= |D|) + // DdC = D/C + // CpRD = C+DdC*D + // E = (A+B*DdC)/CpRD + // F = (B-A*DdC)/CpRD + // else + // CdD = C/D + // DpRC= D+CdD*C + // E = (A*CdD+B)/DpRC + // F = (B*CdD-A)/DpRC // AARCH64-FASTMATH-LABEL: @div_float_cc([2 x float] noundef nofpclass(nan inf) alignstack(8) %a.coerce, [2 x float] noundef nofpclass(nan inf) alignstack(8) %b.coerce) - // - // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[BD:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[ACpBD:%.*]] = fadd fast float - // - // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast float - // - // AARCH64-FASTMATH: [[BC:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast float - // AARCH64-FASTMATH: [[BCmAD:%.*]] = fsub fast float - // - // AARCH64-FASTMATH: fdiv fast float - // AARCH64-FASTMATH: fdiv fast float - // AARCH64-FASTMATH: ret + // |C| + // AARCH64-FASTMATH: call {{.*}}float @llvm.fabs.f32(float {{.*}}) + // |D| + // AARCH64-FASTMATH-NEXT: call {{.*}}float @llvm.fabs.f32(float {{.*}}) + // AARCH64-FASTMATH-NEXT: fcmp {{.*}}ugt float + // AARCH64-FASTMATH-NEXT: br i1 {{.*}}, label + // AARCH64-FASTMATH: abs_rhsr_greater_or_equal_abs_rhsi: + + // |C| >= |D| + // DdC=D/C + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // CpRD=C+CdC*D + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + + // A+BR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // B-AR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fsub {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: abs_rhsr_less_than_abs_rhsi: + + // |C| < |D| + // CdD=C/D + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // DpRC=D+CdD*C + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + + // (A*CdD+B)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fadd {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // (BCdD-A)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}float + // AARCH64-FASTMATH-NEXT: fsub {{.*}}float + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}float + + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: complex_div: + // AARCH64-FASTMATH-NEXT: phi {{.*}}float + // AARCH64-FASTMATH-NEXT: phi {{.*}}float return a / b; } @@ -312,24 +400,68 @@ double _Complex div_double_rc(double a, double _Complex b) { // SPIR: call spir_func {{.*}} @__divdc3( - // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD)) + // a / b = (A+iB) / (C+iD) = (E+iF) + // if (|C| >= |D|) + // DdC = D/C + // CpRD = C+DdC*D + // E = (A+B*DdC)/CpRD + // F = (B-A*DdC)/CpRD + // else + // CdD = C/D + // DpRC= D+CdD*C + // E = (A*CdD+B)/DpRC + // F = (B*CdD-A)/DpRC // AARCH64-FASTMATH-LABEL: @div_double_rc(double noundef nofpclass(nan inf) %a, [2 x double] noundef nofpclass(nan inf) alignstack(8) %b.coerce) - // A = a - // B = 0 - // - // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast double - // BD = 0 - // ACpBD = AC - // - // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast double - // - // BC = 0 - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast double - // - // AARCH64-FASTMATH: fdiv fast double - // AARCH64-FASTMATH: fdiv fast double + // |C| + // AARCH64-FASTMATH: call {{.*}}double @llvm.fabs.f64(double {{.*}}) + // |D| + // AARCH64-FASTMATH-NEXT: call {{.*}}double @llvm.fabs.f64(double {{.*}}) + // AARCH64-FASTMATH-NEXT: fcmp {{.*}}ugt double + // AARCH64-FASTMATH-NEXT: br i1 {{.*}}, label + // AARCH64-FASTMATH: abs_rhsr_greater_or_equal_abs_rhsi: + + // |C| >= |D| + // DdC=D/C + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // CpRD=C+CdC*D + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + + // A+BR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // B-AR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fsub {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: abs_rhsr_less_than_abs_rhsi: + + // |C| < |D| + // CdD=C/D + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // DpRC=D+CdD*C + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + + // (A*CdD+B)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // (BCdD-A)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fsub {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: complex_div: + // AARCH64-FASTMATH-NEXT: phi {{.*}}double + // AARCH64-FASTMATH-NEXT: phi {{.*}}double // AARCH64-FASTMATH: ret return a / b; } @@ -341,23 +473,68 @@ double _Complex div_double_cc(double _Complex a, double _Complex b) { // SPIR: call spir_func {{.*}} @__divdc3( - // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD)) + // a / b = (A+iB) / (C+iD) = (E+iF) + // if (|C| >= |D|) + // DdC = D/C + // CpRD = C+DdC*D + // E = (A+B*DdC)/CpRD + // F = (B-A*DdC)/CpRD + // else + // CdD = C/D + // DpRC= D+CdD*C + // E = (A*CdD+B)/DpRC + // F = (B*CdD-A)/DpRC // AARCH64-FASTMATH-LABEL: @div_double_cc([2 x double] noundef nofpclass(nan inf) alignstack(8) %a.coerce, [2 x double] noundef nofpclass(nan inf) alignstack(8) %b.coerce) - // - // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[BD:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[ACpBD:%.*]] = fadd fast double - // - // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast double - // - // AARCH64-FASTMATH: [[BC:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast double - // AARCH64-FASTMATH: [[BCmAD:%.*]] = fsub fast double - // - // AARCH64-FASTMATH: fdiv fast double - // AARCH64-FASTMATH: fdiv fast double + // |C| + // AARCH64-FASTMATH: call {{.*}}double @llvm.fabs.f64(double {{.*}}) + // |D| + // AARCH64-FASTMATH-NEXT: call {{.*}}double @llvm.fabs.f64(double {{.*}}) + // AARCH64-FASTMATH-NEXT: fcmp {{.*}}ugt double + // AARCH64-FASTMATH-NEXT: br i1 {{.*}}, label + // AARCH64-FASTMATH: abs_rhsr_greater_or_equal_abs_rhsi: + + // |C| >= |D| + // DdC=D/C + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // CpRD=C+CdC*D + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + + // A+BR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // B-AR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fsub {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: abs_rhsr_less_than_abs_rhsi: + + // |C| < |D| + // CdD=C/D + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // DpRC=D+CdD*C + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + + // (A*CdD+B)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fadd {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // (BCdD-A)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}double + // AARCH64-FASTMATH-NEXT: fsub {{.*}}double + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}double + + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: complex_div: + // AARCH64-FASTMATH-NEXT: phi {{.*}}double + // AARCH64-FASTMATH-NEXT: phi {{.*}}double // AARCH64-FASTMATH: ret return a / b; } @@ -505,24 +682,68 @@ long double _Complex div_long_double_rc(long double a, long double _Complex b) { // PPC: ret // SPIR: call spir_func {{.*}} @__divdc3( - // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD)) + // a / b = (A+iB) / (C+iD) = (E+iF) + // if (|C| >= |D|) + // DdC = D/C + // CpRD = C+DdC*D + // E = (A+B*DdC)/CpRD + // F = (B-A*DdC)/CpRD + // else + // CdD = C/D + // DpRC= D+CdD*C + // E = (A*CdD+B)/DpRC + // F = (B*CdD-A)/DpRC // AARCH64-FASTMATH-LABEL: @div_long_double_rc(fp128 noundef nofpclass(nan inf) %a, [2 x fp128] noundef nofpclass(nan inf) alignstack(16) %b.coerce) - // A = a - // B = 0 - // - // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast fp128 - // BD = 0 - // ACpBD = AC - // - // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast fp128 - // - // BC = 0 - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast fp128 - // - // AARCH64-FASTMATH: fdiv fast fp128 - // AARCH64-FASTMATH: fdiv fast fp128 + // |C| + // AARCH64-FASTMATH: call {{.*}}fp128 @llvm.fabs.f128(fp128 {{.*}}) + // |D| + // AARCH64-FASTMATH-NEXT: call {{.*}}fp128 @llvm.fabs.f128(fp128 {{.*}}) + // AARCH64-FASTMATH-NEXT: fcmp {{.*}}ugt fp128 + // AARCH64-FASTMATH-NEXT: br i1 {{.*}}, label + // AARCH64-FASTMATH: abs_rhsr_greater_or_equal_abs_rhsi: + + // |C| >= |D| + // DdC=D/C + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // CpRD=C+CdC*D + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + + // A+BR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // B-AR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fsub {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: abs_rhsr_less_than_abs_rhsi: + + // |C| < |D| + // CdD=C/D + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // DpRC=D+CdD*C + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + + // (A*CdD+B)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // (BCdD-A)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fsub {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: complex_div: + // AARCH64-FASTMATH-NEXT: phi {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: phi {{.*}}fp128 // AARCH64-FASTMATH: ret return a / b; } @@ -537,23 +758,68 @@ long double _Complex div_long_double_cc(long double _Complex a, long double _Com // PPC: ret // SPIR: call spir_func {{.*}} @__divdc3( - // a / b = (A+iB) / (C+iD) = ((AC+BD)/(CC+DD)) + i((BC-AD)/(CC+DD)) + // a / b = (A+iB) / (C+iD) = (E+iF) + // if (|C| >= |D|) + // DdC = D/C + // CpRD = C+DdC*D + // E = (A+B*DdC)/CpRD + // F = (B-A*DdC)/CpRD + // else + // CdD = C/D + // DpRC= D+CdD*C + // E = (A*CdD+B)/DpRC + // F = (B*CdD-A)/DpRC // AARCH64-FASTMATH-LABEL: @div_long_double_cc([2 x fp128] noundef nofpclass(nan inf) alignstack(16) %a.coerce, [2 x fp128] noundef nofpclass(nan inf) alignstack(16) %b.coerce) - // - // AARCH64-FASTMATH: [[AC:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[BD:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[ACpBD:%.*]] = fadd fast fp128 - // - // AARCH64-FASTMATH: [[CC:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[DD:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[CCpDD:%.*]] = fadd fast fp128 - // - // AARCH64-FASTMATH: [[BC:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[AD:%.*]] = fmul fast fp128 - // AARCH64-FASTMATH: [[BCmAD:%.*]] = fsub fast fp128 - // - // AARCH64-FASTMATH: fdiv fast fp128 - // AARCH64-FASTMATH: fdiv fast fp128 + // |C| + // AARCH64-FASTMATH: call {{.*}}fp128 @llvm.fabs.f128(fp128 {{.*}}) + // |D| + // AARCH64-FASTMATH-NEXT: call {{.*}}fp128 @llvm.fabs.f128(fp128 {{.*}}) + // AARCH64-FASTMATH-NEXT: fcmp {{.*}}ugt fp128 + // AARCH64-FASTMATH-NEXT: br i1 {{.*}}, label + // AARCH64-FASTMATH: abs_rhsr_greater_or_equal_abs_rhsi: + + // |C| >= |D| + // DdC=D/C + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // CpRD=C+CdC*D + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + + // A+BR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // B-AR/CpRD + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fsub {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: abs_rhsr_less_than_abs_rhsi: + + // |C| < |D| + // CdD=C/D + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // DpRC=D+CdD*C + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + + // (A*CdD+B)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fadd {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // (BCdD-A)/DpRC + // AARCH64-FASTMATH-NEXT: fmul {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fsub {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: fdiv {{.*}}fp128 + + // AARCH64-FASTMATH-NEXT: br label + // AARCH64-FASTMATH: complex_div: + // AARCH64-FASTMATH-NEXT: phi {{.*}}fp128 + // AARCH64-FASTMATH-NEXT: phi {{.*}}fp128 // AARCH64-FASTMATH: ret return a / b; } diff --git a/clang/test/CodeGen/cx-complex-range.c b/clang/test/CodeGen/cx-complex-range.c new file mode 100644 index 000000000000..8368fa611335 --- /dev/null +++ b/clang/test/CodeGen/cx-complex-range.c @@ -0,0 +1,108 @@ +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=limited -o - | FileCheck %s --check-prefix=LMTD + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -fno-cx-limited-range -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=fortran -o - | FileCheck %s --check-prefix=FRTRN + +// Fast math +// RUN: %clang_cc1 -triple x86_64-unknown-linux-gnu \ +// RUN: -ffast-math -complex-range=limited -emit-llvm -o - %s \ +// RUN: | FileCheck %s --check-prefix=LMTD-FAST + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -fno-cx-fortran-rules -o - | FileCheck %s --check-prefix=FULL + +_Complex float div(_Complex float a, _Complex float b) { + // LABEL: define {{.*}} @div( + // FULL: call {{.*}} @__divsc3 + + // LMTD: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fadd float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fadd float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fsub float + // LMTD-NEXT: fdiv float + // LMTD-NEXT: fdiv float + + // FRTRN: call {{.*}}float @llvm.fabs.f32(float {{.*}}) + // FRTRN-NEXT: call {{.*}}float @llvm.fabs.f32(float {{.*}}) + // FRTRN-NEXT: fcmp {{.*}}ugt float + // FRTRN-NEXT: br i1 {{.*}}, label + // FRTRN: abs_rhsr_greater_or_equal_abs_rhsi: + // FRTRN-NEXT: fdiv {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fadd {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fadd {{.*}}float + // FRTRN-NEXT: fdiv {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fsub {{.*}}float + // FRTRN-NEXT: fdiv {{.*}}float + // FRTRN-NEXT: br label + // FRTRN: abs_rhsr_less_than_abs_rhsi: + // FRTRN-NEXT: fdiv {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fadd {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fadd {{.*}}float + // FRTRN-NEXT: fdiv {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fsub {{.*}}float + // FRTRN-NEXT: fdiv {{.*}}float + // FRTRN-NEXT: br label + // FRTRN: complex_div: + // FRTRN-NEXT: phi {{.*}}float + // FRTRN-NEXT: phi {{.*}}float + + // LMTD-FAST: fmul {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fadd {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fadd {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fsub {{.*}} float + // LMTD-FAST-NEXT: fdiv {{.*}} float + // LMTD-FAST-NEXT: fdiv {{.*}} float + + return a / b; +} + +_Complex float mul(_Complex float a, _Complex float b) { + // LABEL: define {{.*}} @mul( + // FULL: call {{.*}} @__mulsc3 + + // LMTD: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fsub float + // LMTD-NEXT: fadd float + + // FRTRN: fmul {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fmul {{.*}}float + // FRTRN-NEXT: fsub {{.*}}float + // FRTRN-NEXT: fadd {{.*}}float + + // LMTD-FAST: fmul {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fmul {{.*}} float + // LMTD-FAST-NEXT: fsub {{.*}} float + // LMTD-FAST-NEXT: fadd {{.*}} float + + return a * b; +} diff --git a/clang/test/CodeGen/pgo-instrumentation.c b/clang/test/CodeGen/pgo-instrumentation.c index a65c6712291b..c01658065497 100644 --- a/clang/test/CodeGen/pgo-instrumentation.c +++ b/clang/test/CodeGen/pgo-instrumentation.c @@ -3,7 +3,7 @@ // Ensure Pass PGOInstrumentationGenPass is invoked. // RUN: %clang_cc1 -O2 -fprofile-instrument=llvm %s -fdebug-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s -check-prefix=CHECK-PGOGENPASS-INVOKED-INSTR-GEN --check-prefix=CHECK-INSTRPROF // CHECK-PGOGENPASS-INVOKED-INSTR-GEN: Running pass: PGOInstrumentationGen on -// CHECK-INSTRPROF: Running pass: InstrProfiling on +// CHECK-INSTRPROF: Running pass: InstrProfilingLoweringPass on // // Ensure Pass PGOInstrumentationGenPass is not invoked. // RUN: %clang_cc1 -O2 -fprofile-instrument=clang %s -fdebug-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s -check-prefix=CHECK-PGOGENPASS-INVOKED-INSTR-GEN-CLANG @@ -11,7 +11,7 @@ // RUN: %clang_cc1 -O2 -fprofile-instrument=clang %s -fdebug-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-CLANG-INSTRPROF // RUN: %clang_cc1 -O0 -fprofile-instrument=clang %s -fdebug-pass-manager -emit-llvm -o - 2>&1 | FileCheck %s --check-prefix=CHECK-CLANG-INSTRPROF -// CHECK-CLANG-INSTRPROF: Running pass: InstrProfiling on +// CHECK-CLANG-INSTRPROF: Running pass: InstrProfilingLoweringPass on // Ensure Pass PGOInstrumentationUsePass is invoked. // RUN: llvm-profdata merge -o %t.profdata %S/Inputs/pgotestir.profraw diff --git a/clang/test/CodeGen/pragma-cx-limited-range.c b/clang/test/CodeGen/pragma-cx-limited-range.c new file mode 100644 index 000000000000..926da8afbee5 --- /dev/null +++ b/clang/test/CodeGen/pragma-cx-limited-range.c @@ -0,0 +1,107 @@ +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=limited -o - | FileCheck --check-prefix=LMTD %s + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -fno-cx-limited-range -o - | FileCheck %s --check-prefix=FULL + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -complex-range=fortran -o - | FileCheck --check-prefix=FRTRN %s + +// RUN: %clang_cc1 %s -O0 -emit-llvm -triple x86_64-unknown-unknown \ +// RUN: -fno-cx-fortran-rules -o - | FileCheck --check-prefix=FULL %s + +_Complex float pragma_on_mul(_Complex float a, _Complex float b) { +#pragma STDC CX_LIMITED_RANGE ON + // LABEL: define {{.*}} @pragma_on_mul( + // FULL: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fsub float + // FULL-NEXT: fadd float + + // LMTD: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fsub float + // LMTD-NEXT: fadd float + + // FRTRN: fmul float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fsub float + // FRTRN-NEXT: fadd float + + return a * b; +} + +_Complex float pragma_off_mul(_Complex float a, _Complex float b) { +#pragma STDC CX_LIMITED_RANGE OFF + // LABEL: define {{.*}} @pragma_off_mul( + // FULL: call {{.*}} @__mulsc3 + + // LMTD: call {{.*}} @__mulsc3 + + // FRTRN: call {{.*}} @__mulsc3 + + return a * b; +} + +_Complex float pragma_on_div(_Complex float a, _Complex float b) { +#pragma STDC CX_LIMITED_RANGE ON + // LABEL: define {{.*}} @pragma_on_div( + // FULL: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fadd float + // FULL-NEXT: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fadd float + // FULL-NEXT: fmul float + // FULL-NEXT: fmul float + // FULL-NEXT: fsub float + // FULL-NEXT: fdiv float + // FULL: fdiv float + + // LMTD: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fadd float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fadd float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fmul float + // LMTD-NEXT: fsub float + // LMTD-NEXT: fdiv float + // LMTD-NEXT: fdiv float + + // FRTRN: fmul float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fadd float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fadd float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fmul float + // FRTRN-NEXT: fsub float + // FRTRN-NEXT: fdiv float + // FRTRN-NEXT: fdiv float + + return a / b; +} + +_Complex float pragma_off_div(_Complex float a, _Complex float b) { +#pragma STDC CX_LIMITED_RANGE OFF + // LABEL: define {{.*}} @pragma_off_div( + // FULL: call {{.*}} @__divsc3 + + // LMTD: call {{.*}} @__divsc3 + + // FRTRN: call {{.*}} @__divsc3 + + return a / b; +} diff --git a/clang/test/CodeGen/scoped-atomic-ops.c b/clang/test/CodeGen/scoped-atomic-ops.c new file mode 100644 index 000000000000..b0032046639b --- /dev/null +++ b/clang/test/CodeGen/scoped-atomic-ops.c @@ -0,0 +1,331 @@ +// RUN: %clang_cc1 %s -emit-llvm -o - -triple=amdgcn-amd-amdhsa -ffreestanding \ +// RUN: -fvisibility=hidden | FileCheck %s + +// CHECK-LABEL: define hidden i32 @fi1a( +// CHECK: [[TMP0:%.*]] = load atomic i32, ptr [[PTR0:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = load atomic i32, ptr [[PTR1:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = load atomic i32, ptr [[PTR2:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = load atomic i32, ptr [[PTR3:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = load atomic i32, ptr [[PTR4:.+]] syncscope("singlethread-one-as") monotonic, align 4 +int fi1a(int *i) { + int v; + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + return v; +} + +// CHECK-LABEL: define hidden i32 @fi1b( +// CHECK: [[TMP0:%.*]] = load atomic i32, ptr [[PTR0:%.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = load atomic i32, ptr [[PTR1:%.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = load atomic i32, ptr [[PTR2:%.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = load atomic i32, ptr [[PTR3:%.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = load atomic i32, ptr [[PTR4:%.+]] syncscope("singlethread-one-as") monotonic, align 4 +// +int fi1b(int *i) { + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + return *i; +} + +// CHECK-LABEL: define hidden void @fi2a( +// CHECK: store atomic i32 [[TMP0:%.+]], ptr [[PTR0:%.+]] syncscope("one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP1:%.+]], ptr [[PTR1:%.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP2:%.+]], ptr [[PTR2:%.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP3:%.+]], ptr [[PTR3:%.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP4:%.+]], ptr [[PTR4:%.+]] syncscope("singlethread-one-as") monotonic, align 4 +// +void fi2a(int *i) { + int v = 1; + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden void @fi2b( +// CHECK: store atomic i32 [[TMP0:%.+]], ptr [[PTR0:%.+]] syncscope("one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP1:%.+]], ptr [[PTR1:%.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP2:%.+]], ptr [[PTR2:%.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP3:%.+]], ptr [[PTR3:%.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: store atomic i32 [[TMP4:%.+]], ptr [[PTR4:%.+]] syncscope("singlethread-one-as") monotonic, align 4 +void fi2b(int *i) { + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden void @fi3a( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("one-as") monotonic, align 4 +void fi3a(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); +} + +// CHECK-LABEL: define hidden void @fi3b( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("agent-one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("agent-one-as") monotonic, align 4 +void fi3b(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); +} + +// CHECK-LABEL: define hidden void @fi3c( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("workgroup-one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("workgroup-one-as") monotonic, align 4 +void fi3c(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); +} + +// CHECK-LABEL: define hidden void @fi3d( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("wavefront-one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("wavefront-one-as") monotonic, align 4 +void fi3d(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); +} + +// CHECK-LABEL: define hidden void @fi3e( +// CHECK: [[TMP0:%.*]] = atomicrmw add ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP1:%.*]] = atomicrmw sub ptr [[PTR1:%.+]], i32 [[VAL1:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP2:%.*]] = atomicrmw and ptr [[PTR2:%.+]], i32 [[VAL2:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP3:%.*]] = atomicrmw or ptr [[PTR3:%.+]], i32 [[VAL3:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP4:%.*]] = atomicrmw xor ptr [[PTR4:%.+]], i32 [[VAL4:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP5:%.*]] = atomicrmw nand ptr [[PTR5:%.+]], i32 [[VAL5:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP6:%.*]] = atomicrmw min ptr [[PTR6:%.+]], i32 [[VAL6:.+]] syncscope("singlethread-one-as") monotonic, align 4 +// CHECK: [[TMP7:%.*]] = atomicrmw max ptr [[PTR7:%.+]], i32 [[VAL7:.+]] syncscope("singlethread-one-as") monotonic, align 4 +void fi3e(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4a( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("one-as") acquire acquire, align 4 +_Bool fi4a(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SYSTEM); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4b( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("agent-one-as") acquire acquire, align 4 +_Bool fi4b(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_DEVICE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4c( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("workgroup-one-as") acquire acquire, align 4 +_Bool fi4c(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_WRKGRP); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4d( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("wavefront-one-as") acquire acquire, align 4 +_Bool fi4d(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_WVFRNT); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi4e( +// CHECK: [[TMP0:%.*]] = cmpxchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("singlethread-one-as") acquire acquire, align 4 +_Bool fi4e(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5a( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("one-as") acquire acquire, align 4 +_Bool fi5a(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n(i, &cmp, 1, 1, __ATOMIC_ACQUIRE, + __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SYSTEM); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5b( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("agent-one-as") acquire acquire, align 4 +_Bool fi5b(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n(i, &cmp, 1, 1, __ATOMIC_ACQUIRE, + __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_DEVICE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5c( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("workgroup-one-as") acquire acquire, align 4 +_Bool fi5c(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n( + i, &cmp, 1, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_WRKGRP); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5d( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("wavefront-one-as") acquire acquire, align 4 +_Bool fi5d(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n( + i, &cmp, 1, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_WVFRNT); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi5e( +// CHECK: [[TMP0:%.*]] = cmpxchg weak ptr [[PTR0:%.+]], i32 [[VAL0:.+]], i32 [[VAL1:.+]] syncscope("singlethread-one-as") acquire acquire, align 4 +_Bool fi5e(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n( + i, &cmp, 1, 1, __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, __MEMORY_SCOPE_SINGLE); +} + +// CHECK-LABEL: define hidden i32 @fi6a( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("one-as") monotonic, align 4 +int fi6a(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return ret; +} + +// CHECK-LABEL: define hidden i32 @fi6b( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("agent-one-as") monotonic, align 4 +int fi6b(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_DEVICE); + return ret; +} + +// CHECK-LABEL: define hidden i32 @fi6c( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("workgroup-one-as") monotonic, align 4 +int fi6c(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_WRKGRP); + return ret; +} + +// CHECK-LABEL: define hidden i32 @fi6d( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("wavefront-one-as") monotonic, align 4 +int fi6d(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_WVFRNT); + return ret; +} + +// CHECK-LABEL: define hidden i32 @fi6e( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i32 [[VAL0:.+]] syncscope("singlethread-one-as") monotonic, align 4 +int fi6e(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_SINGLE); + return ret; +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7a( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("one-as") monotonic, align 1 +_Bool fi7a(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_SYSTEM); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7b( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("agent-one-as") monotonic, align 1 +_Bool fi7b(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_DEVICE); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7c( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("workgroup-one-as") monotonic, align 1 +_Bool fi7c(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_WRKGRP); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7d( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("wavefront-one-as") monotonic, align 1 +_Bool fi7d(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_WVFRNT); +} + +// CHECK-LABEL: define hidden zeroext i1 @fi7e( +// CHECK: [[TMP0:%.*]] = atomicrmw xchg ptr [[PTR0:%.+]], i8 [[VAL0:.+]] syncscope("singlethread-one-as") monotonic, align 1 +_Bool fi7e(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_SINGLE); +} diff --git a/clang/test/CodeGen/tbaa.c b/clang/test/CodeGen/tbaa.c new file mode 100644 index 000000000000..0ab81f60a719 --- /dev/null +++ b/clang/test/CodeGen/tbaa.c @@ -0,0 +1,116 @@ +// RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -no-struct-path-tbaa -disable-llvm-passes %s -emit-llvm -o - | FileCheck %s +// RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -disable-llvm-passes %s -emit-llvm -o - | FileCheck %s -check-prefixes=PATH +// RUN: %clang_cc1 -triple x86_64-apple-darwin -O0 -disable-llvm-passes %s -emit-llvm -o - | FileCheck %s -check-prefix=NO-TBAA +// RUN: %clang_cc1 -triple x86_64-apple-darwin -O1 -relaxed-aliasing -disable-llvm-passes %s -emit-llvm -o - | FileCheck %s -check-prefix=NO-TBAA +// Test TBAA metadata generated by front-end. +// +// NO-TBAA-NOT: !tbaa + +typedef unsigned char uint8_t; +typedef unsigned short uint16_t; +typedef unsigned int uint32_t; +typedef unsigned long long uint64_t; + +typedef enum { + RED_AUTO_32, + GREEN_AUTO_32, + BLUE_AUTO_32 +} EnumAuto32; + +typedef enum { + RED_AUTO_64, + GREEN_AUTO_64, + BLUE_AUTO_64 = 0x100000000ull +} EnumAuto64; + +typedef enum : uint16_t { + RED_16, + GREEN_16, + BLUE_16 +} Enum16; + +typedef enum : uint8_t { + RED_8, + GREEN_8, + BLUE_8 +} Enum8; + +uint32_t g0(EnumAuto32 *E, uint32_t *val) { +// CHECK-LABEL: define{{.*}} i32 @g0( +// CHECK: store i32 5, ptr %{{.*}}, align 4, !tbaa [[TAG_i32:!.*]] +// CHECK: store i32 0, ptr %{{.*}}, align 4, !tbaa [[TAG_i32]] +// CHECK: load i32, ptr %{{.*}}, align 4, !tbaa [[TAG_i32]] +// PATH-LABEL: define{{.*}} i32 @g0( +// PATH: store i32 5, ptr %{{.*}}, align 4, !tbaa [[TAG_i32:!.*]] +// PATH: store i32 0, ptr %{{.*}}, align 4, !tbaa [[TAG_i32]] +// PATH: load i32, ptr %{{.*}}, align 4, !tbaa [[TAG_i32]] + *val = 5; + *E = RED_AUTO_32; + return *val; +} + +uint64_t g1(EnumAuto64 *E, uint64_t *val) { +// CHECK-LABEL: define{{.*}} i64 @g1( +// CHECK: store i64 5, ptr %{{.*}}, align 8, !tbaa [[TAG_i64:!.*]] +// CHECK: store i64 0, ptr %{{.*}}, align 8, !tbaa [[TAG_long:!.*]] +// CHECK: load i64, ptr %{{.*}}, align 8, !tbaa [[TAG_i64]] +// PATH-LABEL: define{{.*}} i64 @g1( +// PATH: store i64 5, ptr %{{.*}}, align 8, !tbaa [[TAG_i64:!.*]] +// PATH: store i64 0, ptr %{{.*}}, align 8, !tbaa [[TAG_long:!.*]] +// PATH: load i64, ptr %{{.*}}, align 8, !tbaa [[TAG_i64]] + *val = 5; + *E = RED_AUTO_64; + return *val; +} + +uint16_t g2(Enum16 *E, uint16_t *val) { +// CHECK-LABEL: define{{.*}} i16 @g2( +// CHECK: store i16 5, ptr %{{.*}}, align 2, !tbaa [[TAG_i16:!.*]] +// CHECK: store i16 0, ptr %{{.*}}, align 2, !tbaa [[TAG_i16]] +// CHECK: load i16, ptr %{{.*}}, align 2, !tbaa [[TAG_i16]] +// PATH-LABEL: define{{.*}} i16 @g2( +// PATH: store i16 5, ptr %{{.*}}, align 2, !tbaa [[TAG_i16:!.*]] +// PATH: store i16 0, ptr %{{.*}}, align 2, !tbaa [[TAG_i16]] +// PATH: load i16, ptr %{{.*}}, align 2, !tbaa [[TAG_i16]] + *val = 5; + *E = RED_16; + return *val; +} + +uint8_t g3(Enum8 *E, uint8_t *val) { +// CHECK-LABEL: define{{.*}} i8 @g3( +// CHECK: store i8 5, ptr %{{.*}}, align 1, !tbaa [[TAG_i8:!.*]] +// CHECK: store i8 0, ptr %{{.*}}, align 1, !tbaa [[TAG_i8]] +// CHECK: load i8, ptr %{{.*}}, align 1, !tbaa [[TAG_i8]] +// PATH-LABEL: define{{.*}} i8 @g3( +// PATH: store i8 5, ptr %{{.*}}, align 1, !tbaa [[TAG_i8:!.*]] +// PATH: store i8 0, ptr %{{.*}}, align 1, !tbaa [[TAG_i8]] +// PATH: load i8, ptr %{{.*}}, align 1, !tbaa [[TAG_i8]] + *val = 5; + *E = RED_8; + return *val; +} + +// CHECK: [[TYPE_char:!.*]] = !{!"omnipotent char", [[TAG_c_tbaa:!.*]], +// CHECK: [[TAG_c_tbaa]] = !{!"Simple C/C++ TBAA"} +// CHECK: [[TAG_i32]] = !{[[TYPE_i32:!.*]], [[TYPE_i32]], i64 0} +// CHECK: [[TYPE_i32]] = !{!"int", [[TYPE_char]], +// CHECK: [[TAG_i64]] = !{[[TYPE_i64:!.*]], [[TYPE_i64]], i64 0} +// CHECK: [[TYPE_i64]] = !{!"long long", [[TYPE_char]], +// CHECK: [[TAG_long]] = !{[[TYPE_long:!.*]], [[TYPE_long]], i64 0} +// CHECK: [[TYPE_long]] = !{!"long", [[TYPE_char]], +// CHECK: [[TAG_i16]] = !{[[TYPE_i16:!.*]], [[TYPE_i16]], i64 0} +// CHECK: [[TYPE_i16]] = !{!"short", [[TYPE_char]], +// CHECK: [[TAG_i8]] = !{[[TYPE_i8:!.*]], [[TYPE_char]], i64 0} + +// PATH: [[TYPE_char:!.*]] = !{!"omnipotent char", [[TAG_c_tbaa:!.*]], +// PATH: [[TAG_c_tbaa]] = !{!"Simple C/C++ TBAA"} +// PATH: [[TAG_i32]] = !{[[TYPE_i32:!.*]], [[TYPE_i32]], i64 0} +// PATH: [[TYPE_i32]] = !{!"int", [[TYPE_char]], +// PATH: [[TAG_i64]] = !{[[TYPE_i64:!.*]], [[TYPE_i64]], i64 0} +// PATH: [[TYPE_i64]] = !{!"long long", [[TYPE_char]], +// PATH: [[TAG_long]] = !{[[TYPE_long:!.*]], [[TYPE_long]], i64 0} +// PATH: [[TYPE_long]] = !{!"long", [[TYPE_char]], +// PATH: [[TAG_i16]] = !{[[TYPE_i16:!.*]], [[TYPE_i16]], i64 0} +// PATH: [[TYPE_i16]] = !{!"short", [[TYPE_char]], +// PATH: [[TAG_i8]] = !{[[TYPE_i8:!.*]], [[TYPE_char]], i64 0} diff --git a/clang/test/CodeGenCUDA/offloading-entries.cu b/clang/test/CodeGenCUDA/offloading-entries.cu index 46235051f1e4..4f5cf65ecd0b 100644 --- a/clang/test/CodeGenCUDA/offloading-entries.cu +++ b/clang/test/CodeGenCUDA/offloading-entries.cu @@ -17,31 +17,47 @@ //. // CUDA: @.omp_offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00" // CUDA: @.omp_offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z18__device_stub__foov, ptr @.omp_offloading.entry_name, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries", align 1 -// CUDA: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [8 x i8] c"_Z3barv\00" -// CUDA: @.omp_offloading.entry._Z3barv = weak constant %struct.__tgt_offload_entry { ptr @_Z18__device_stub__barv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries", align 1 -// CUDA: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [2 x i8] c"x\00" -// CUDA: @.omp_offloading.entry.x = weak constant %struct.__tgt_offload_entry { ptr @x, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries", align 1 +// CUDA: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00" +// CUDA: @.omp_offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z21__device_stub__kernelv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries", align 1 +// CUDA: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00" +// CUDA: @.omp_offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries", align 1 +// CUDA: @.omp_offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00" +// CUDA: @.omp_offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.omp_offloading.entry_name.3, i64 4, i32 2, i32 1 }, section "cuda_offloading_entries", align 1 +// CUDA: @.omp_offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00" +// CUDA: @.omp_offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.omp_offloading.entry_name.4, i64 4, i32 3, i32 1 }, section "cuda_offloading_entries", align 1 //. // HIP: @.omp_offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00" // HIP: @.omp_offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z3foov, ptr @.omp_offloading.entry_name, i64 0, i32 0, i32 0 }, section "hip_offloading_entries", align 1 -// HIP: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [8 x i8] c"_Z3barv\00" -// HIP: @.omp_offloading.entry._Z3barv = weak constant %struct.__tgt_offload_entry { ptr @_Z3barv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "hip_offloading_entries", align 1 -// HIP: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [2 x i8] c"x\00" -// HIP: @.omp_offloading.entry.x = weak constant %struct.__tgt_offload_entry { ptr @x, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "hip_offloading_entries", align 1 +// HIP: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00" +// HIP: @.omp_offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z6kernelv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "hip_offloading_entries", align 1 +// HIP: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00" +// HIP: @.omp_offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "hip_offloading_entries", align 1 +// HIP: @.omp_offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00" +// HIP: @.omp_offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.omp_offloading.entry_name.3, i64 4, i32 2, i32 1 }, section "hip_offloading_entries", align 1 +// HIP: @.omp_offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00" +// HIP: @.omp_offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.omp_offloading.entry_name.4, i64 4, i32 3, i32 1 }, section "hip_offloading_entries", align 1 //. // CUDA-COFF: @.omp_offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00" // CUDA-COFF: @.omp_offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z18__device_stub__foov, ptr @.omp_offloading.entry_name, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1 -// CUDA-COFF: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [8 x i8] c"_Z3barv\00" -// CUDA-COFF: @.omp_offloading.entry._Z3barv = weak constant %struct.__tgt_offload_entry { ptr @_Z18__device_stub__barv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1 -// CUDA-COFF: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [2 x i8] c"x\00" -// CUDA-COFF: @.omp_offloading.entry.x = weak constant %struct.__tgt_offload_entry { ptr @x, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1 +// CUDA-COFF: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00" +// CUDA-COFF: @.omp_offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z21__device_stub__kernelv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1 +// CUDA-COFF: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00" +// CUDA-COFF: @.omp_offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "cuda_offloading_entries$OE", align 1 +// CUDA-COFF: @.omp_offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00" +// CUDA-COFF: @.omp_offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.omp_offloading.entry_name.3, i64 4, i32 2, i32 1 }, section "cuda_offloading_entries$OE", align 1 +// CUDA-COFF: @.omp_offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00" +// CUDA-COFF: @.omp_offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.omp_offloading.entry_name.4, i64 4, i32 3, i32 1 }, section "cuda_offloading_entries$OE", align 1 //. // HIP-COFF: @.omp_offloading.entry_name = internal unnamed_addr constant [8 x i8] c"_Z3foov\00" // HIP-COFF: @.omp_offloading.entry._Z3foov = weak constant %struct.__tgt_offload_entry { ptr @_Z3foov, ptr @.omp_offloading.entry_name, i64 0, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1 -// HIP-COFF: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [8 x i8] c"_Z3barv\00" -// HIP-COFF: @.omp_offloading.entry._Z3barv = weak constant %struct.__tgt_offload_entry { ptr @_Z3barv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1 -// HIP-COFF: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [2 x i8] c"x\00" -// HIP-COFF: @.omp_offloading.entry.x = weak constant %struct.__tgt_offload_entry { ptr @x, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1 +// HIP-COFF: @.omp_offloading.entry_name.1 = internal unnamed_addr constant [11 x i8] c"_Z6kernelv\00" +// HIP-COFF: @.omp_offloading.entry._Z6kernelv = weak constant %struct.__tgt_offload_entry { ptr @_Z6kernelv, ptr @.omp_offloading.entry_name.1, i64 0, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1 +// HIP-COFF: @.omp_offloading.entry_name.2 = internal unnamed_addr constant [4 x i8] c"var\00" +// HIP-COFF: @.omp_offloading.entry.var = weak constant %struct.__tgt_offload_entry { ptr @var, ptr @.omp_offloading.entry_name.2, i64 4, i32 0, i32 0 }, section "hip_offloading_entries$OE", align 1 +// HIP-COFF: @.omp_offloading.entry_name.3 = internal unnamed_addr constant [5 x i8] c"surf\00" +// HIP-COFF: @.omp_offloading.entry.surf = weak constant %struct.__tgt_offload_entry { ptr @surf, ptr @.omp_offloading.entry_name.3, i64 4, i32 2, i32 1 }, section "hip_offloading_entries$OE", align 1 +// HIP-COFF: @.omp_offloading.entry_name.4 = internal unnamed_addr constant [4 x i8] c"tex\00" +// HIP-COFF: @.omp_offloading.entry.tex = weak constant %struct.__tgt_offload_entry { ptr @tex, ptr @.omp_offloading.entry_name.4, i64 4, i32 3, i32 1 }, section "hip_offloading_entries$OE", align 1 //. // CUDA-LABEL: @_Z18__device_stub__foov( // CUDA-NEXT: entry: @@ -72,34 +88,52 @@ // HIP-COFF-NEXT: ret void // __global__ void foo() {} +__device__ int var = 1; +const __device__ int constant = 1; +extern __device__ int external; -// CUDA-LABEL: @_Z18__device_stub__barv( +// CUDA-LABEL: @_Z21__device_stub__kernelv( // CUDA-NEXT: entry: -// CUDA-NEXT: [[TMP0:%.*]] = call i32 @cudaLaunch(ptr @_Z18__device_stub__barv) +// CUDA-NEXT: [[TMP0:%.*]] = call i32 @cudaLaunch(ptr @_Z21__device_stub__kernelv) // CUDA-NEXT: br label [[SETUP_END:%.*]] // CUDA: setup.end: // CUDA-NEXT: ret void // -// HIP-LABEL: @_Z18__device_stub__barv( +// HIP-LABEL: @_Z21__device_stub__kernelv( // HIP-NEXT: entry: -// HIP-NEXT: [[TMP0:%.*]] = call i32 @hipLaunchByPtr(ptr @_Z3barv) +// HIP-NEXT: [[TMP0:%.*]] = call i32 @hipLaunchByPtr(ptr @_Z6kernelv) // HIP-NEXT: br label [[SETUP_END:%.*]] // HIP: setup.end: // HIP-NEXT: ret void // -// CUDA-COFF-LABEL: @_Z18__device_stub__barv( +// CUDA-COFF-LABEL: @_Z21__device_stub__kernelv( // CUDA-COFF-NEXT: entry: -// CUDA-COFF-NEXT: [[TMP0:%.*]] = call i32 @cudaLaunch(ptr @_Z18__device_stub__barv) +// CUDA-COFF-NEXT: [[TMP0:%.*]] = call i32 @cudaLaunch(ptr @_Z21__device_stub__kernelv) // CUDA-COFF-NEXT: br label [[SETUP_END:%.*]] // CUDA-COFF: setup.end: // CUDA-COFF-NEXT: ret void // -// HIP-COFF-LABEL: @_Z18__device_stub__barv( +// HIP-COFF-LABEL: @_Z21__device_stub__kernelv( // HIP-COFF-NEXT: entry: -// HIP-COFF-NEXT: [[TMP0:%.*]] = call i32 @hipLaunchByPtr(ptr @_Z3barv) +// HIP-COFF-NEXT: [[TMP0:%.*]] = call i32 @hipLaunchByPtr(ptr @_Z6kernelv) // HIP-COFF-NEXT: br label [[SETUP_END:%.*]] // HIP-COFF: setup.end: // HIP-COFF-NEXT: ret void // -__global__ void bar() {} -__device__ int x = 1; +__global__ void kernel() { external = 1; } + +struct surfaceReference { int desc; }; + +template +struct __attribute__((device_builtin_surface_type)) surface : public surfaceReference {}; + +surface surf; + +struct textureReference { + int desc; +}; + +template +struct __attribute__((device_builtin_texture_type)) texture : public textureReference {}; + +texture tex; diff --git a/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl b/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl index 77091f8390a1..a70e224b81e4 100644 --- a/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl +++ b/clang/test/CodeGenHLSL/builtins/RWBuffer-annotations.hlsl @@ -16,9 +16,9 @@ void main() { } // CHECK: !hlsl.uavs = !{![[Single:[0-9]+]], ![[Array:[0-9]+]], ![[SingleAllocated:[0-9]+]], ![[ArrayAllocated:[0-9]+]], ![[SingleSpace:[0-9]+]], ![[ArraySpace:[0-9]+]]} -// CHECK-DAG: ![[Single]] = !{ptr @"?Buffer1@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i32 -1, i32 0} -// CHECK-DAG: ![[Array]] = !{ptr @"?BufferArray@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i32 -1, i32 0} -// CHECK-DAG: ![[SingleAllocated]] = !{ptr @"?Buffer2@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i32 3, i32 0} -// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @"?BufferArray2@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i32 4, i32 0} -// CHECK-DAG: ![[SingleSpace]] = !{ptr @"?Buffer3@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i32 3, i32 1} -// CHECK-DAG: ![[ArraySpace]] = !{ptr @"?BufferArray3@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i32 4, i32 1} +// CHECK-DAG: ![[Single]] = !{ptr @"?Buffer1@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i1 false, i32 -1, i32 0} +// CHECK-DAG: ![[Array]] = !{ptr @"?BufferArray@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i1 false, i32 -1, i32 0} +// CHECK-DAG: ![[SingleAllocated]] = !{ptr @"?Buffer2@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i1 false, i32 3, i32 0} +// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @"?BufferArray2@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i1 false, i32 4, i32 0} +// CHECK-DAG: ![[SingleSpace]] = !{ptr @"?Buffer3@@3V?$RWBuffer@M@hlsl@@A", !"RWBuffer", i32 10, i1 false, i32 3, i32 1} +// CHECK-DAG: ![[ArraySpace]] = !{ptr @"?BufferArray3@@3PAV?$RWBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RWBuffer >", i32 10, i1 false, i32 4, i32 1} diff --git a/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl new file mode 100644 index 000000000000..ce7d84ecf5b1 --- /dev/null +++ b/clang/test/CodeGenHLSL/builtins/RasterizerOrderedBuffer-annotations.hlsl @@ -0,0 +1,20 @@ +// RUN: %clang_cc1 -triple dxil-pc-shadermodel6.0-pixel -x hlsl -emit-llvm -disable-llvm-passes -o - %s | FileCheck %s + +RasterizerOrderedBuffer Buffer1; +RasterizerOrderedBuffer > BufferArray[4]; + +RasterizerOrderedBuffer Buffer2 : register(u3); +RasterizerOrderedBuffer > BufferArray2[4] : register(u4); + +RasterizerOrderedBuffer Buffer3 : register(u3, space1); +RasterizerOrderedBuffer > BufferArray3[4] : register(u4, space1); + +void main() {} + +// CHECK: !hlsl.uavs = !{![[Single:[0-9]+]], ![[Array:[0-9]+]], ![[SingleAllocated:[0-9]+]], ![[ArrayAllocated:[0-9]+]], ![[SingleSpace:[0-9]+]], ![[ArraySpace:[0-9]+]]} +// CHECK-DAG: ![[Single]] = !{ptr @"?Buffer1@@3V?$RasterizerOrderedBuffer@M@hlsl@@A", !"RasterizerOrderedBuffer", i32 10, i1 true, i32 -1, i32 0} +// CHECK-DAG: ![[Array]] = !{ptr @"?BufferArray@@3PAV?$RasterizerOrderedBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RasterizerOrderedBuffer >", i32 10, i1 true, i32 -1, i32 0} +// CHECK-DAG: ![[SingleAllocated]] = !{ptr @"?Buffer2@@3V?$RasterizerOrderedBuffer@M@hlsl@@A", !"RasterizerOrderedBuffer", i32 10, i1 true, i32 3, i32 0} +// CHECK-DAG: ![[ArrayAllocated]] = !{ptr @"?BufferArray2@@3PAV?$RasterizerOrderedBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RasterizerOrderedBuffer >", i32 10, i1 true, i32 4, i32 0} +// CHECK-DAG: ![[SingleSpace]] = !{ptr @"?Buffer3@@3V?$RasterizerOrderedBuffer@M@hlsl@@A", !"RasterizerOrderedBuffer", i32 10, i1 true, i32 3, i32 1} +// CHECK-DAG: ![[ArraySpace]] = !{ptr @"?BufferArray3@@3PAV?$RasterizerOrderedBuffer@T?$__vector@M$03@__clang@@@hlsl@@A", !"RasterizerOrderedBuffer >", i32 10, i1 true, i32 4, i32 1} diff --git a/clang/test/CodeGenHLSL/cbuf.hlsl b/clang/test/CodeGenHLSL/cbuf.hlsl index 92c883943d03..5dee1feb902a 100644 --- a/clang/test/CodeGenHLSL/cbuf.hlsl +++ b/clang/test/CodeGenHLSL/cbuf.hlsl @@ -24,5 +24,5 @@ float foo() { // CHECK: !hlsl.cbufs = !{![[CBMD:[0-9]+]]} // CHECK: !hlsl.srvs = !{![[TBMD:[0-9]+]]} -// CHECK: ![[CBMD]] = !{ptr @[[CB]], !"A.cb.ty", i32 13, i32 0, i32 2} -// CHECK: ![[TBMD]] = !{ptr @[[TB]], !"A.tb.ty", i32 15, i32 2, i32 1} +// CHECK: ![[CBMD]] = !{ptr @[[CB]], !"A.cb.ty", i32 13, i1 false, i32 0, i32 2} +// CHECK: ![[TBMD]] = !{ptr @[[TB]], !"A.tb.ty", i32 15, i1 false, i32 2, i32 1} diff --git a/clang/test/Driver/aarch64-outliner.c b/clang/test/Driver/aarch64-outliner.c index 42e43b433e28..06e5de11ec49 100644 --- a/clang/test/Driver/aarch64-outliner.c +++ b/clang/test/Driver/aarch64-outliner.c @@ -1,7 +1,9 @@ // REQUIRES: aarch64-registered-target // RUN: %clang --target=aarch64 -moutline -S %s -### 2>&1 | FileCheck %s -check-prefix=ON +// RUN: %clang --target=aarch64_be -moutline -S %s -### 2>&1 | FileCheck %s -check-prefix=ON // ON: "-mllvm" "-enable-machine-outliner" // RUN: %clang --target=aarch64 -moutline -mno-outline -S %s -### 2>&1 | FileCheck %s -check-prefix=OFF +// RUN: %clang --target=aarch64_be -moutline -mno-outline -S %s -### 2>&1 | FileCheck %s -check-prefix=OFF // OFF: "-mllvm" "-enable-machine-outliner=never" // RUN: %clang --target=x86_64 -moutline -S %s -### 2>&1 | FileCheck %s -check-prefix=WARN // WARN: warning: 'x86_64' does not support '-moutline'; flag ignored [-Woption-ignored] diff --git a/clang/test/Driver/arm-cortex-cpus-2.c b/clang/test/Driver/arm-cortex-cpus-2.c index 5b52488bc498..c322303d2278 100644 --- a/clang/test/Driver/arm-cortex-cpus-2.c +++ b/clang/test/Driver/arm-cortex-cpus-2.c @@ -562,8 +562,11 @@ // RUN: %clang -target arm -mcpu=cortex-m85 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-M85 %s // CHECK-CORTEX-M85: "-cc1"{{.*}} "-triple" "thumbv8.1m.main-{{.*}} "-target-cpu" "cortex-m85" +// RUN: %clang -target arm -mcpu=cortex-m52 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CORTEX-M52 %s +// CHECK-CORTEX-M52: "-cc1"{{.*}} "-triple" "thumbv8.1m.main-{{.*}} "-target-cpu" "cortex-m52" + // RUN: %clang -target arm -mcpu=neoverse-n2 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-NEOVERSE-N2 %s -// CHECK-NEOVERSE-N2: "-cc1"{{.*}} "-triple" "armv8.5a-{{.*}}" "-target-cpu" "neoverse-n2" +// CHECK-NEOVERSE-N2: "-cc1"{{.*}} "-triple" "armv9a-{{.*}}" "-target-cpu" "neoverse-n2" // ================== Check whether -mcpu accepts mixed-case values. // RUN: %clang -target arm-linux-gnueabi -mcpu=Cortex-a5 -### -c %s 2>&1 | FileCheck -check-prefix=CHECK-CASE-INSENSITIVE-CPUV7A %s diff --git a/clang/test/Driver/cl-options.c b/clang/test/Driver/cl-options.c index 6d929b19e7e2..81d1b907eced 100644 --- a/clang/test/Driver/cl-options.c +++ b/clang/test/Driver/cl-options.c @@ -747,7 +747,7 @@ // Validate that the default triple is used when run an empty tools dir is specified // RUN: %clang_cl -vctoolsdir "" -### -- %s 2>&1 | FileCheck %s --check-prefix VCTOOLSDIR -// VCTOOLSDIR: "-triple" "{{[a-zA-Z0-9_-]*}}-pc-windows-msvc19.20.0" +// VCTOOLSDIR: "-triple" "{{[a-zA-Z0-9_-]*}}-pc-windows-msvc19.33.0" // Validate that built-in include paths are based on the supplied path // RUN: %clang_cl --target=aarch64-pc-windows-msvc -vctoolsdir "/fake" -winsdkdir "/foo" -winsdkversion 10.0.12345.0 -### -- %s 2>&1 | FileCheck %s --check-prefix FAKEDIR @@ -787,7 +787,7 @@ // RUN: %clang_cl -vctoolsdir "" /arm64EC /c -### -- %s 2>&1 | FileCheck --check-prefix=ARM64EC %s // ARM64EC-NOT: /arm64EC has been overridden by specified target -// ARM64EC: "-triple" "arm64ec-pc-windows-msvc19.20.0" +// ARM64EC: "-triple" "arm64ec-pc-windows-msvc19.33.0" // RUN: %clang_cl -vctoolsdir "" /arm64EC /c -target x86_64-pc-windows-msvc -### -- %s 2>&1 | FileCheck --check-prefix=ARM64EC_OVERRIDE %s // ARM64EC_OVERRIDE: warning: /arm64EC has been overridden by specified target: x86_64-pc-windows-msvc; option ignored diff --git a/clang/test/Driver/clang-offload-bundler-zlib.c b/clang/test/Driver/clang-offload-bundler-zlib.c index c46c32a4a053..a57ee6da9a86 100644 --- a/clang/test/Driver/clang-offload-bundler-zlib.c +++ b/clang/test/Driver/clang-offload-bundler-zlib.c @@ -55,14 +55,15 @@ // Check unbundling archive. // // RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \ -// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%T/hip_bundle1.bc -compress +// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%t.hip_bundle1.bc -compress // RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \ -// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%T/hip_bundle2.bc -compress -// RUN: llvm-ar cr %T/hip_archive.a %T/hip_bundle1.bc %T/hip_bundle2.bc +// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%t.hip_bundle2.bc -compress +// RUN: rm -f %t.hip_archive.a +// RUN: llvm-ar cr %t.hip_archive.a %t.hip_bundle1.bc %t.hip_bundle2.bc // RUN: clang-offload-bundler -unbundle -type=a -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \ -// RUN: -output=%T/hip_900.a -output=%T/hip_906.a -input=%T/hip_archive.a -// RUN: llvm-ar t %T/hip_900.a | FileCheck -check-prefix=HIP-AR-900 %s -// RUN: llvm-ar t %T/hip_906.a | FileCheck -check-prefix=HIP-AR-906 %s +// RUN: -output=%t.hip_900.a -output=%t.hip_906.a -input=%t.hip_archive.a +// RUN: llvm-ar t %t.hip_900.a | FileCheck -check-prefix=HIP-AR-900 %s +// RUN: llvm-ar t %t.hip_906.a | FileCheck -check-prefix=HIP-AR-906 %s // HIP-AR-900-DAG: hip_bundle1-hip-amdgcn-amd-amdhsa--gfx900 // HIP-AR-900-DAG: hip_bundle2-hip-amdgcn-amd-amdhsa--gfx900 // HIP-AR-906-DAG: hip_bundle1-hip-amdgcn-amd-amdhsa--gfx906 diff --git a/clang/test/Driver/clang-offload-bundler-zstd.c b/clang/test/Driver/clang-offload-bundler-zstd.c index b2b588b72d4d..3b577d4d166a 100644 --- a/clang/test/Driver/clang-offload-bundler-zstd.c +++ b/clang/test/Driver/clang-offload-bundler-zstd.c @@ -48,18 +48,18 @@ // RUN: diff %t.tgt1 %t.res.tgt1 // RUN: diff %t.tgt2 %t.res.tgt2 -// // Check unbundling archive. // // RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \ -// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%T/hip_bundle1.bc -compress +// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%t.hip_bundle1.bc -compress // RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \ -// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%T/hip_bundle2.bc -compress -// RUN: llvm-ar cr %T/hip_archive.a %T/hip_bundle1.bc %T/hip_bundle2.bc +// RUN: -input=%t.tgt1 -input=%t.tgt2 -output=%t.hip_bundle2.bc -compress +// RUN: rm -f %t.hip_archive.a +// RUN: llvm-ar cr %t.hip_archive.a %t.hip_bundle1.bc %t.hip_bundle2.bc // RUN: clang-offload-bundler -unbundle -type=a -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \ -// RUN: -output=%T/hip_900.a -output=%T/hip_906.a -input=%T/hip_archive.a -// RUN: llvm-ar t %T/hip_900.a | FileCheck -check-prefix=HIP-AR-900 %s -// RUN: llvm-ar t %T/hip_906.a | FileCheck -check-prefix=HIP-AR-906 %s +// RUN: -output=%t.hip_900.a -output=%t.hip_906.a -input=%t.hip_archive.a +// RUN: llvm-ar t %t.hip_900.a | FileCheck -check-prefix=HIP-AR-900 %s +// RUN: llvm-ar t %t.hip_906.a | FileCheck -check-prefix=HIP-AR-906 %s // HIP-AR-900-DAG: hip_bundle1-hip-amdgcn-amd-amdhsa--gfx900 // HIP-AR-900-DAG: hip_bundle2-hip-amdgcn-amd-amdhsa--gfx900 // HIP-AR-906-DAG: hip_bundle1-hip-amdgcn-amd-amdhsa--gfx906 diff --git a/clang/test/Driver/fdefine-target-os-macros.c b/clang/test/Driver/fdefine-target-os-macros.c new file mode 100644 index 000000000000..030d4ce34cb2 --- /dev/null +++ b/clang/test/Driver/fdefine-target-os-macros.c @@ -0,0 +1,250 @@ +// RUN: %clang -### --target=arm64-apple-darwin %s 2>&1 | FileCheck %s --check-prefix=DARWIN-DEFAULT +// DARWIN-DEFAULT-NOT: "-fdefine-target-os-macros" + +// RUN: %clang -### --target=arm-none-linux-gnu %s 2>&1 | FileCheck %s --check-prefix=NON-DARWIN-DEFAULT +// RUN: %clang -### --target=x86_64-pc-win32 %s 2>&1 | FileCheck %s --check-prefix=NON-DARWIN-DEFAULT +// NON-DARWIN-DEFAULT-NOT: "-fdefine-target-os-macros" + +// RUN: %clang -dM -E --target=arm64-apple-macos \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=1 \ +// RUN: -DIPHONE=0 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-ios \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=1 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=1 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-ios-macabi \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=1 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=1 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-ios-simulator \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=1 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=1 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-tvos \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=1 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=1 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-tvos-simulator \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=1 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=1 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-watchos \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=1 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=1 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-watchos-simulator \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=1 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=1 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=1 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=arm64-apple-driverkit \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=1 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=0 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=1 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=x86_64-pc-linux-gnu \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=0 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=0 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=1 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=x86_64-pc-win32 \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=0 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=0 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=1 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=x86_64-pc-windows-gnu \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=0 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=0 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=1 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=0 + +// RUN: %clang -dM -E --target=sparc-none-solaris \ +// RUN: -fdefine-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s -DMAC=0 \ +// RUN: -DOSX=0 \ +// RUN: -DIPHONE=0 \ +// RUN: -DIOS=0 \ +// RUN: -DTV=0 \ +// RUN: -DWATCH=0 \ +// RUN: -DDRIVERKIT=0 \ +// RUN: -DMACCATALYST=0 \ +// RUN: -DEMBEDDED=0 \ +// RUN: -DSIMULATOR=0 \ +// RUN: -DWINDOWS=0 \ +// RUN: -DLINUX=0 \ +// RUN: -DUNIX=1 + +// RUN: %clang -dM -E --target=arm64-apple-macos \ +// RUN: -fno-define-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s --check-prefix=NEG + +// RUN: %clang -dM -E --target=arm64-apple-macos \ +// RUN: -fdefine-target-os-macros \ +// RUN: -fno-define-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s --check-prefix=NEG + +// RUN: %clang -dM -E --target=x86_64-pc-windows \ +// RUN: -fdefine-target-os-macros \ +// RUN: -fno-define-target-os-macros %s 2>&1 \ +// RUN: | FileCheck %s --check-prefix=NEG + +// NEG-NOT: #define TARGET_OS_ + +// CHECK-DAG: #define TARGET_OS_MAC [[MAC]] +// CHECK-DAG: #define TARGET_OS_OSX [[OSX]] +// CHECK-DAG: #define TARGET_OS_IPHONE [[IPHONE]] +// CHECK-DAG: #define TARGET_OS_IOS [[IOS]] +// CHECK-DAG: #define TARGET_OS_TV [[TV]] +// CHECK-DAG: #define TARGET_OS_WATCH [[WATCH]] +// CHECK-DAG: #define TARGET_OS_DRIVERKIT [[DRIVERKIT]] +// CHECK-DAG: #define TARGET_OS_MACCATALYST [[MACCATALYST]] +// CHECK-DAG: #define TARGET_OS_SIMULATOR [[SIMULATOR]] +// Deprecated +// CHECK-DAG: #define TARGET_OS_EMBEDDED [[EMBEDDED]] +// CHECK-DAG: #define TARGET_OS_NANO [[WATCH]] +// CHECK-DAG: #define TARGET_IPHONE_SIMULATOR [[SIMULATOR]] +// CHECK-DAG: #define TARGET_OS_UIKITFORMAC [[MACCATALYST]] +// Non-darwin OSes +// CHECK-DAG: #define TARGET_OS_WIN32 [[WINDOWS]] +// CHECK-DAG: #define TARGET_OS_WINDOWS [[WINDOWS]] +// CHECK-DAG: #define TARGET_OS_LINUX [[LINUX]] +// CHECK-DAG: #define TARGET_OS_UNIX [[UNIX]] diff --git a/clang/test/Driver/hip-offload-compress-zlib.hip b/clang/test/Driver/hip-offload-compress-zlib.hip index a29b6d037350..7557fdde8786 100644 --- a/clang/test/Driver/hip-offload-compress-zlib.hip +++ b/clang/test/Driver/hip-offload-compress-zlib.hip @@ -4,13 +4,13 @@ // Test compress bundled bitcode. -// RUN: rm -rf %T/a.bc +// RUN: rm -rf %t.bc // RUN: %clang -c -v --target=x86_64-linux-gnu \ // RUN: -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \ // RUN: -fgpu-rdc -nogpuinc -nogpulib \ // RUN: %S/Inputs/hip_multiple_inputs/a.cu \ // RUN: --offload-compress --offload-device-only --gpu-bundle-output \ -// RUN: -o %T/a.bc \ +// RUN: -o %t.bc \ // RUN: 2>&1 | FileCheck %s // CHECK: clang-offload-bundler{{.*}} -type=bc @@ -23,7 +23,7 @@ // RUN: %clang --hip-link -### -v --target=x86_64-linux-gnu \ // RUN: --offload-arch=gfx1100 --offload-arch=gfx1101 \ // RUN: -fgpu-rdc -nogpulib \ -// RUN: %T/a.bc --offload-device-only \ +// RUN: %t.bc --offload-device-only \ // RUN: 2>&1 | FileCheck -check-prefix=UNBUNDLE %s // UNBUNDLE: clang-offload-bundler{{.*}} "-type=bc" diff --git a/clang/test/Driver/hip-offload-compress-zstd.hip b/clang/test/Driver/hip-offload-compress-zstd.hip index 688c2c85329c..3680ae47974a 100644 --- a/clang/test/Driver/hip-offload-compress-zstd.hip +++ b/clang/test/Driver/hip-offload-compress-zstd.hip @@ -4,13 +4,13 @@ // Test compress bundled bitcode. -// RUN: rm -rf %T/a.bc +// RUN: rm -rf %t.bc // RUN: %clang -c -v --target=x86_64-linux-gnu \ // RUN: -x hip --offload-arch=gfx1100 --offload-arch=gfx1101 \ // RUN: -fgpu-rdc -nogpuinc -nogpulib \ // RUN: %S/Inputs/hip_multiple_inputs/a.cu \ // RUN: --offload-compress --offload-device-only --gpu-bundle-output \ -// RUN: -o %T/a.bc \ +// RUN: -o %t.bc \ // RUN: 2>&1 | FileCheck %s // CHECK: clang-offload-bundler{{.*}} -type=bc @@ -23,7 +23,7 @@ // RUN: %clang --hip-link -### -v --target=x86_64-linux-gnu \ // RUN: --offload-arch=gfx1100 --offload-arch=gfx1101 \ // RUN: -fgpu-rdc -nogpulib \ -// RUN: %T/a.bc --offload-device-only \ +// RUN: %t.bc --offload-device-only \ // RUN: 2>&1 | FileCheck -check-prefix=UNBUNDLE %s // UNBUNDLE: clang-offload-bundler{{.*}} "-type=bc" diff --git a/clang/test/Driver/linker-wrapper-image.c b/clang/test/Driver/linker-wrapper-image.c index 73d3c40810c3..a2a1996f6643 100644 --- a/clang/test/Driver/linker-wrapper-image.c +++ b/clang/test/Driver/linker-wrapper-image.c @@ -80,24 +80,33 @@ // CUDA-NEXT: br i1 icmp ne (ptr @__start_cuda_offloading_entries, ptr @__stop_cuda_offloading_entries), label %while.entry, label %while.end // CUDA: while.entry: -// CUDA-NEXT: %entry1 = phi ptr [ @__start_cuda_offloading_entries, %entry ], [ %7, %if.end ] -// CUDA-NEXT: %1 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 0 -// CUDA-NEXT: %addr = load ptr, ptr %1, align 8 -// CUDA-NEXT: %2 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 1 -// CUDA-NEXT: %name = load ptr, ptr %2, align 8 -// CUDA-NEXT: %3 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 2 -// CUDA-NEXT: %size = load i64, ptr %3, align 4 -// CUDA-NEXT: %4 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 3 -// CUDA-NEXT: %flag = load i32, ptr %4, align 4 -// CUDA-NEXT: %5 = icmp eq i64 %size, 0 -// CUDA-NEXT: br i1 %5, label %if.then, label %if.else +// CUDA-NEXT: %entry1 = phi ptr [ @__start_cuda_offloading_entries, %entry ], [ %11, %if.end ] +// CUDA-NEXT: %1 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 0 +// CUDA-NEXT: %addr = load ptr, ptr %1, align 8 +// CUDA-NEXT: %2 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 1 +// CUDA-NEXT: %name = load ptr, ptr %2, align 8 +// CUDA-NEXT: %3 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 2 +// CUDA-NEXT: %size = load i64, ptr %3, align 4 +// CUDA-NEXT: %4 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 3 +// CUDA-NEXT: %flags = load i32, ptr %4, align 4 +// CUDA-NEXT: %5 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 4 +// CUDA-NEXT: %textype = load i32, ptr %5, align 4 +// CUDA-NEXT: %type = and i32 %flags, 7 +// CUDA-NEXT: %6 = and i32 %flags, 8 +// CUDA-NEXT: %extern = lshr i32 %6, 3 +// CUDA-NEXT: %7 = and i32 %flags, 16 +// CUDA-NEXT: %constant = lshr i32 %7, 4 +// CUDA-NEXT: %8 = and i32 %flags, 32 +// CUDA-NEXT: %normalized = lshr i32 %8, 5 +// CUDA-NEXT: %9 = icmp eq i64 %size, 0 +// CUDA-NEXT: br i1 %9, label %if.then, label %if.else // CUDA: if.then: -// CUDA-NEXT: %6 = call i32 @__cudaRegisterFunction(ptr %0, ptr %addr, ptr %name, ptr %name, i32 -1, ptr null, ptr null, ptr null, ptr null, ptr null) +// CUDA-NEXT: %10 = call i32 @__cudaRegisterFunction(ptr %0, ptr %addr, ptr %name, ptr %name, i32 -1, ptr null, ptr null, ptr null, ptr null, ptr null) // CUDA-NEXT: br label %if.end // CUDA: if.else: -// CUDA-NEXT: switch i32 %flag, label %if.end [ +// CUDA-NEXT: switch i32 %type, label %if.end [ // CUDA-NEXT: i32 0, label %sw.global // CUDA-NEXT: i32 1, label %sw.managed // CUDA-NEXT: i32 2, label %sw.surface @@ -105,22 +114,24 @@ // CUDA-NEXT: ] // CUDA: sw.global: -// CUDA-NEXT: call void @__cudaRegisterVar(ptr %0, ptr %addr, ptr %name, ptr %name, i32 0, i64 %size, i32 0, i32 0) +// CUDA-NEXT: call void @__cudaRegisterVar(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %extern, i64 %size, i32 %constant, i32 0) // CUDA-NEXT: br label %if.end // CUDA: sw.managed: // CUDA-NEXT: br label %if.end // CUDA: sw.surface: +// CUDA-NEXT: call void @__cudaRegisterSurface(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %textype, i32 %extern) // CUDA-NEXT: br label %if.end // CUDA: sw.texture: +// CUDA-NEXT: call void @__cudaRegisterTexture(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %textype, i32 %normalized, i32 %extern) // CUDA-NEXT: br label %if.end // CUDA: if.end: -// CUDA-NEXT: %7 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 1 -// CUDA-NEXT: %8 = icmp eq ptr %7, @__stop_cuda_offloading_entries -// CUDA-NEXT: br i1 %8, label %while.end, label %while.entry +// CUDA-NEXT: %11 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 1 +// CUDA-NEXT: %12 = icmp eq ptr %11, @__stop_cuda_offloading_entries +// CUDA-NEXT: br i1 %12, label %while.end, label %while.entry // CUDA: while.end: // CUDA-NEXT: ret void @@ -168,7 +179,7 @@ // HIP-NEXT: br i1 icmp ne (ptr @__start_hip_offloading_entries, ptr @__stop_hip_offloading_entries), label %while.entry, label %while.end // HIP: while.entry: -// HIP-NEXT: %entry1 = phi ptr [ @__start_hip_offloading_entries, %entry ], [ %7, %if.end ] +// HIP-NEXT: %entry1 = phi ptr [ @__start_hip_offloading_entries, %entry ], [ %11, %if.end ] // HIP-NEXT: %1 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 0 // HIP-NEXT: %addr = load ptr, ptr %1, align 8 // HIP-NEXT: %2 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 1 @@ -176,16 +187,25 @@ // HIP-NEXT: %3 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 2 // HIP-NEXT: %size = load i64, ptr %3, align 4 // HIP-NEXT: %4 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 3 -// HIP-NEXT: %flag = load i32, ptr %4, align 4 -// HIP-NEXT: %5 = icmp eq i64 %size, 0 -// HIP-NEXT: br i1 %5, label %if.then, label %if.else +// HIP-NEXT: %flags = load i32, ptr %4, align 4 +// HIP-NEXT: %5 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 0, i32 4 +// HIP-NEXT: %textype = load i32, ptr %5, align 4 +// HIP-NEXT: %type = and i32 %flags, 7 +// HIP-NEXT: %6 = and i32 %flags, 8 +// HIP-NEXT: %extern = lshr i32 %6, 3 +// HIP-NEXT: %7 = and i32 %flags, 16 +// HIP-NEXT: %constant = lshr i32 %7, 4 +// HIP-NEXT: %8 = and i32 %flags, 32 +// HIP-NEXT: %normalized = lshr i32 %8, 5 +// HIP-NEXT: %9 = icmp eq i64 %size, 0 +// HIP-NEXT: br i1 %9, label %if.then, label %if.else // HIP: if.then: -// HIP-NEXT: %6 = call i32 @__hipRegisterFunction(ptr %0, ptr %addr, ptr %name, ptr %name, i32 -1, ptr null, ptr null, ptr null, ptr null, ptr null) +// HIP-NEXT: %10 = call i32 @__hipRegisterFunction(ptr %0, ptr %addr, ptr %name, ptr %name, i32 -1, ptr null, ptr null, ptr null, ptr null, ptr null) // HIP-NEXT: br label %if.end // HIP: if.else: -// HIP-NEXT: switch i32 %flag, label %if.end [ +// HIP-NEXT: switch i32 %type, label %if.end [ // HIP-NEXT: i32 0, label %sw.global // HIP-NEXT: i32 1, label %sw.managed // HIP-NEXT: i32 2, label %sw.surface @@ -193,22 +213,24 @@ // HIP-NEXT: ] // HIP: sw.global: -// HIP-NEXT: call void @__hipRegisterVar(ptr %0, ptr %addr, ptr %name, ptr %name, i32 0, i64 %size, i32 0, i32 0) +// HIP-NEXT: call void @__hipRegisterVar(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %extern, i64 %size, i32 %constant, i32 0) // HIP-NEXT: br label %if.end // HIP: sw.managed: // HIP-NEXT: br label %if.end // HIP: sw.surface: +// HIP-NEXT: call void @__hipRegisterSurface(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %textype, i32 %extern) // HIP-NEXT: br label %if.end // HIP: sw.texture: +// HIP-NEXT: call void @__hipRegisterTexture(ptr %0, ptr %addr, ptr %name, ptr %name, i32 %textype, i32 %normalized, i32 %extern) // HIP-NEXT: br label %if.end // HIP: if.end: -// HIP-NEXT: %7 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 1 -// HIP-NEXT: %8 = icmp eq ptr %7, @__stop_hip_offloading_entries -// HIP-NEXT: br i1 %8, label %while.end, label %while.entry +// HIP-NEXT: %11 = getelementptr inbounds %struct.__tgt_offload_entry, ptr %entry1, i64 1 +// HIP-NEXT: %12 = icmp eq ptr %11, @__stop_hip_offloading_entries +// HIP-NEXT: br i1 %12, label %while.end, label %while.entry // HIP: while.end: // HIP-NEXT: ret void diff --git a/clang/test/Driver/linker-wrapper.c b/clang/test/Driver/linker-wrapper.c index e82febd61823..b763a003452b 100644 --- a/clang/test/Driver/linker-wrapper.c +++ b/clang/test/Driver/linker-wrapper.c @@ -123,8 +123,8 @@ // RUN: --linker-path=/usr/bin/ld --device-linker=a --device-linker=nvptx64-nvidia-cuda=b -- \ // RUN: %t.o -o a.out 2>&1 | FileCheck %s --check-prefix=LINKER-ARGS -// LINKER-ARGS: clang{{.*}}--target=amdgcn-amd-amdhsa{{.*}}-Wl,a -// LINKER-ARGS: clang{{.*}}--target=nvptx64-nvidia-cuda{{.*}}-Wl,a -Wl,b +// LINKER-ARGS: clang{{.*}}--target=amdgcn-amd-amdhsa{{.*}}a +// LINKER-ARGS: clang{{.*}}--target=nvptx64-nvidia-cuda{{.*}}a b // RUN: not clang-linker-wrapper --dry-run --host-triple=x86_64-unknown-linux-gnu -ldummy \ // RUN: --linker-path=/usr/bin/ld --device-linker=a --device-linker=nvptx64-nvidia-cuda=b -- \ diff --git a/clang/test/Driver/range.c b/clang/test/Driver/range.c new file mode 100644 index 000000000000..8d456a997d69 --- /dev/null +++ b/clang/test/Driver/range.c @@ -0,0 +1,39 @@ +// Test range options for complex multiplication and division. + +// RUN: %clang -### -target x86_64 -fcx-limited-range -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=LMTD %s + +// RUN: %clang -### -target x86_64 -fno-cx-limited-range -c %s 2>&1 \ +// RUN: | FileCheck %s + +// RUN: %clang -### -target x86_64 -fcx-fortran-rules -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=FRTRN %s + +// RUN: %clang -### -target x86_64 -fno-cx-fortran-rules -c %s 2>&1 \ +// RUN: | FileCheck %s + +// RUN: %clang -### -target x86_64 -fcx-limited-range \ +// RUN: -fcx-fortran-rules -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=WARN1 %s + +// RUN: %clang -### -target x86_64 -fcx-fortran-rules \ +// RUN: -fcx-limited-range -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=WARN2 %s + +// RUN: %clang -### -target x86_64 -ffast-math -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=LMTD %s + +// RUN: %clang -### -target x86_64 -ffast-math -fcx-limited-range -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=LMTD %s + +// RUN: %clang -### -target x86_64 -fcx-limited-range -ffast-math -c %s 2>&1 \ +// RUN: | FileCheck --check-prefix=LMTD %s + +// LMTD: -complex-range=limited +// LMTD-NOT: -complex-range=fortran +// CHECK-NOT: -complex-range=limited +// FRTRN: -complex-range=fortran +// FRTRN-NOT: -complex-range=limited +// CHECK-NOT: -complex-range=fortran +// WARN1: warning: overriding '-fcx-limited-range' option with '-fcx-fortran-rules' [-Woverriding-option] +// WARN2: warning: overriding '-fcx-fortran-rules' option with '-fcx-limited-range' [-Woverriding-option] diff --git a/clang/test/Driver/rocm-detect.hip b/clang/test/Driver/rocm-detect.hip index 947c4f995be1..3644f215a345 100644 --- a/clang/test/Driver/rocm-detect.hip +++ b/clang/test/Driver/rocm-detect.hip @@ -78,39 +78,39 @@ // RUN: | FileCheck -check-prefixes=ROCM-ENV,HIP-PATH %s // Test detecting latest /opt/rocm-{release} directory. -// RUN: rm -rf %T/opt -// RUN: mkdir -p %T/opt -// RUN: cp -r %S/Inputs/rocm %T/opt/rocm-3.9.0-1234 -// RUN: cp -r %S/Inputs/rocm %T/opt/rocm-3.10.0 -// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=gfx1010 --sysroot=%T \ +// RUN: rm -rf %t/opt +// RUN: mkdir -p %t/opt +// RUN: cp -r %S/Inputs/rocm %t/opt/rocm-3.9.0-1234 +// RUN: cp -r %S/Inputs/rocm %t/opt/rocm-3.10.0 +// RUN: %clang -### --target=x86_64-linux-gnu --offload-arch=gfx1010 --sysroot=%t \ // RUN: --print-rocm-search-dirs %s 2>&1 \ // RUN: | FileCheck -check-prefixes=ROCM-REL %s -// Test ROCm installation built by SPACK by invoke clang at %T/rocm-spack/llvm-amdgpu-* +// Test ROCm installation built by SPACK by invoke clang at %t/rocm-spack/llvm-amdgpu-* // directory through a soft link. -// RUN: rm -rf %T/rocm-spack -// RUN: cp -r %S/Inputs/rocm-spack %T -// RUN: ln -fs %clang %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -// RUN: %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -### -v \ -// RUN: -resource-dir=%T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/lib/clang \ +// RUN: rm -rf %t/rocm-spack +// RUN: cp -r %S/Inputs/rocm-spack %t +// RUN: ln -fs %clang %t/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang +// RUN: %t/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -### -v \ +// RUN: -resource-dir=%t/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/lib/clang \ // RUN: -target x86_64-linux-gnu --cuda-gpu-arch=gfx900 --print-rocm-search-dirs %s 2>&1 \ // RUN: | FileCheck -check-prefixes=SPACK %s // Test SPACK installation with multiple hip and rocm-device-libs packages of the same // ROCm release. --hip-path and --rocm-device-lib-path can be used to specify them. -// RUN: cp -r %T/rocm-spack/hip-* %T/rocm-spack/hip-4.0.0-abcd -// RUN: %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -### -v \ +// RUN: cp -r %t/rocm-spack/hip-* %t/rocm-spack/hip-4.0.0-abcd +// RUN: %t/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang -### -v \ // RUN: -target x86_64-linux-gnu --cuda-gpu-arch=gfx900 \ -// RUN: --hip-path=%T/rocm-spack/hip-4.0.0-abcd \ +// RUN: --hip-path=%t/rocm-spack/hip-4.0.0-abcd \ // RUN: %s 2>&1 | FileCheck -check-prefixes=SPACK-SET %s // Test invalid SPACK ROCm installation missing hip and rocm-device-libs packages. -// RUN: rm -rf %T/rocm-spack/hip-* -// RUN: rm -rf %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/amdgcn -// RUN: %T/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang --version 2>&1 \ +// RUN: rm -rf %t/rocm-spack/hip-* +// RUN: rm -rf %t/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/amdgcn +// RUN: %t/rocm-spack/llvm-amdgpu-4.0.0-ieagcs7inf7runpyfvepqkurasoglq4z/bin/clang --version 2>&1 \ // RUN: | FileCheck -check-prefixes=SPACK-MISS-SILENT %s // GFX902-DEFAULTLIBS: error: cannot find ROCm device library for gfx902; provide its path via '--rocm-path' or '--rocm-device-lib-path', or pass '-nogpulib' to build without ROCm device library diff --git a/clang/test/ExtractAPI/language.c b/clang/test/ExtractAPI/language.c index 6facd18f5d98..fe98626c8461 100644 --- a/clang/test/ExtractAPI/language.c +++ b/clang/test/ExtractAPI/language.c @@ -4,29 +4,42 @@ // RUN: %t/c.reference.output.json.in >> %t/c.reference.output.json // RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}@g" \ // RUN: %t/objc.reference.output.json.in >> %t/objc.reference.output.json +// RUN: sed -e "s@INPUT_DIR@%{/t:regex_replacement}@g" \ +// RUN: %t/objcpp.reference.output.json.in >> %t/objcpp.reference.output.json -// RUN: %clang -extract-api -x c-header -target arm64-apple-macosx \ +// RUN: %clang_cc1 -extract-api -x c-header -triple arm64-apple-macosx \ // RUN: %t/c.h -o %t/c.output.json | FileCheck -allow-empty %s -// RUN: %clang -extract-api -x objective-c-header -target arm64-apple-macosx \ +// RUN: %clang_cc1 -extract-api -x objective-c-header -triple arm64-apple-macosx \ // RUN: %t/objc.h -o %t/objc.output.json | FileCheck -allow-empty %s +// RUN: %clang_cc1 -extract-api -x objective-c++-header -triple arm64-apple-macosx \ +// RUN: %t/objcpp.h -o %t/objcpp.output.json | FileCheck -allow-empty %s // Generator version is not consistent across test runs, normalize it. // RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \ // RUN: %t/c.output.json >> %t/c.output-normalized.json // RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \ // RUN: %t/objc.output.json >> %t/objc.output-normalized.json +// RUN: sed -e "s@\"generator\": \".*\"@\"generator\": \"?\"@g" \ +// RUN: %t/objcpp.output.json >> %t/objcpp.output-normalized.json // RUN: diff %t/c.reference.output.json %t/c.output-normalized.json // RUN: diff %t/objc.reference.output.json %t/objc.output-normalized.json +// RUN: diff %t/objcpp.reference.output.json %t/objcpp.output-normalized.json // CHECK-NOT: error: // CHECK-NOT: warning: //--- c.h char c; +///expected-no-diagnostics //--- objc.h char objc; +///expected-no-diagnostics + +//--- objcpp.h +char objcpp; +///expected-no-diagnostics //--- c.reference.output.json.in { @@ -196,3 +209,87 @@ char objc; } ] } +//--- objcpp.reference.output.json.in +{ + "metadata": { + "formatVersion": { + "major": 0, + "minor": 5, + "patch": 3 + }, + "generator": "?" + }, + "module": { + "name": "", + "platform": { + "architecture": "arm64", + "operatingSystem": { + "minimumVersion": { + "major": 11, + "minor": 0, + "patch": 0 + }, + "name": "macosx" + }, + "vendor": "apple" + } + }, + "relationships": [], + "symbols": [ + { + "accessLevel": "public", + "declarationFragments": [ + { + "kind": "typeIdentifier", + "preciseIdentifier": "c:C", + "spelling": "char" + }, + { + "kind": "text", + "spelling": " " + }, + { + "kind": "identifier", + "spelling": "objcpp" + }, + { + "kind": "text", + "spelling": ";" + } + ], + "identifier": { + "interfaceLanguage": "objective-c++", + "precise": "c:@objcpp" + }, + "kind": { + "displayName": "Global Variable", + "identifier": "objective-c++.var" + }, + "location": { + "position": { + "character": 5, + "line": 0 + }, + "uri": "file://INPUT_DIR/objcpp.h" + }, + "names": { + "navigator": [ + { + "kind": "identifier", + "spelling": "objcpp" + } + ], + "subHeading": [ + { + "kind": "identifier", + "spelling": "objcpp" + } + ], + "title": "objcpp" + }, + "pathComponents": [ + "objcpp" + ] + } + ] +} diff --git a/clang/test/Layout/ms-x86-declspec-empty_bases.cpp b/clang/test/Layout/ms-x86-declspec-empty_bases.cpp index cc13a980cb5d..4738ce5720f7 100644 --- a/clang/test/Layout/ms-x86-declspec-empty_bases.cpp +++ b/clang/test/Layout/ms-x86-declspec-empty_bases.cpp @@ -264,3 +264,63 @@ int _ = sizeof(G); // CHECK-NEXT: | [sizeof=12, align=4, // CHECK-NEXT: | nvsize=12, nvalign=4] } + +namespace test5 { + +struct A { + int a; +}; +struct B { + int b; +}; +struct C {}; +struct __declspec(align(16)) D {}; +struct E { + [[msvc::no_unique_address]] C c; +}; +struct __declspec(empty_bases) X : A, D, B, C, E { +}; + +// CHECK: *** Dumping AST Record Layout +// CHECK-NEXT: 0 | struct test5::A +// CHECK-NEXT: 0 | int a +// CHECK-NEXT: | [sizeof=4, align=4, +// CHECK-NEXT: | nvsize=4, nvalign=4] + +// CHECK: *** Dumping AST Record Layout +// CHECK-NEXT: 0 | struct test5::D (empty) +// CHECK-NEXT: | [sizeof=16, align=16, +// CHECK-NEXT: | nvsize=0, nvalign=16] + +// CHECK: *** Dumping AST Record Layout +// CHECK-NEXT: 0 | struct test5::B +// CHECK-NEXT: 0 | int b +// CHECK-NEXT: | [sizeof=4, align=4, +// CHECK-NEXT: | nvsize=4, nvalign=4] + +// CHECK: *** Dumping AST Record Layout +// CHECK-NEXT: 0 | struct test5::C (empty) +// CHECK-NEXT: | [sizeof=1, align=1, +// CHECK-NEXT: | nvsize=0, nvalign=1] + +// CHECK: *** Dumping AST Record Layout +// CHECK-NEXT: 0 | struct test5::E (empty) +// CHECK-NEXT: 0 | struct test5::C c (empty) +// CHECK-NEXT: | [sizeof=1, align=1, +// CHECK-NEXT: | nvsize=1, nvalign=1] + +// CHECK: *** Dumping AST Record Layout +// CHECK-NEXT: 0 | struct test5::X +// CHECK-NEXT: 0 | struct test5::A (base) +// CHECK-NEXT: 0 | int a +// CHECK-NEXT: 0 | struct test5::D (base) (empty) +// CHECK-NEXT: 0 | struct test5::C (base) (empty) +// CHECK-NEXT: 4 | struct test5::B (base) +// CHECK-NEXT: 4 | int b +// CHECK-NEXT: 8 | struct test5::E (base) (empty) +// CHECK-NEXT: 8 | struct test5::C c (empty) +// CHECK-NEXT: | [sizeof=16, align=16, +// CHECK-NEXT: | nvsize=16, nvalign=16] + +int _ = sizeof(X); +} diff --git a/clang/test/Misc/pragma-attribute-supported-attributes-list.test b/clang/test/Misc/pragma-attribute-supported-attributes-list.test index 707fc8875089..bdfda430eea8 100644 --- a/clang/test/Misc/pragma-attribute-supported-attributes-list.test +++ b/clang/test/Misc/pragma-attribute-supported-attributes-list.test @@ -90,6 +90,7 @@ // CHECK-NEXT: LoaderUninitialized (SubjectMatchRule_variable_is_global) // CHECK-NEXT: Lockable (SubjectMatchRule_record) // CHECK-NEXT: MIGServerRoutine (SubjectMatchRule_function, SubjectMatchRule_objc_method, SubjectMatchRule_block) +// CHECK-NEXT: MSConstexpr (SubjectMatchRule_function) // CHECK-NEXT: MSStruct (SubjectMatchRule_record) // CHECK-NEXT: MaybeUndef (SubjectMatchRule_variable_is_parameter) // CHECK-NEXT: MicroMips (SubjectMatchRule_function) diff --git a/clang/test/Misc/target-invalid-cpu-note.c b/clang/test/Misc/target-invalid-cpu-note.c index c7146e63add5..e840a9208f5a 100644 --- a/clang/test/Misc/target-invalid-cpu-note.c +++ b/clang/test/Misc/target-invalid-cpu-note.c @@ -1,7 +1,7 @@ // Use CHECK-NEXT instead of multiple CHECK-SAME to ensure we will fail if there is anything extra in the output. // RUN: not %clang_cc1 -triple armv5--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix ARM // ARM: error: unknown target CPU 'not-a-cpu' -// ARM-NEXT: note: valid target CPU values are: arm8, arm810, strongarm, strongarm110, strongarm1100, strongarm1110, arm7tdmi, arm7tdmi-s, arm710t, arm720t, arm9, arm9tdmi, arm920, arm920t, arm922t, arm940t, ep9312, arm10tdmi, arm1020t, arm9e, arm946e-s, arm966e-s, arm968e-s, arm10e, arm1020e, arm1022e, arm926ej-s, arm1136j-s, arm1136jf-s, mpcore, mpcorenovfp, arm1176jz-s, arm1176jzf-s, arm1156t2-s, arm1156t2f-s, cortex-m0, cortex-m0plus, cortex-m1, sc000, cortex-a5, cortex-a7, cortex-a8, cortex-a9, cortex-a12, cortex-a15, cortex-a17, krait, cortex-r4, cortex-r4f, cortex-r5, cortex-r7, cortex-r8, cortex-r52, sc300, cortex-m3, cortex-m4, cortex-m7, cortex-m23, cortex-m33, cortex-m35p, cortex-m55, cortex-m85, cortex-a32, cortex-a35, cortex-a53, cortex-a55, cortex-a57, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78c, cortex-a710, cortex-x1, cortex-x1c, neoverse-n1, neoverse-n2, neoverse-v1, cyclone, exynos-m3, exynos-m4, exynos-m5, kryo, iwmmxt, xscale, swift{{$}} +// ARM-NEXT: note: valid target CPU values are: arm8, arm810, strongarm, strongarm110, strongarm1100, strongarm1110, arm7tdmi, arm7tdmi-s, arm710t, arm720t, arm9, arm9tdmi, arm920, arm920t, arm922t, arm940t, ep9312, arm10tdmi, arm1020t, arm9e, arm946e-s, arm966e-s, arm968e-s, arm10e, arm1020e, arm1022e, arm926ej-s, arm1136j-s, arm1136jf-s, mpcore, mpcorenovfp, arm1176jz-s, arm1176jzf-s, arm1156t2-s, arm1156t2f-s, cortex-m0, cortex-m0plus, cortex-m1, sc000, cortex-a5, cortex-a7, cortex-a8, cortex-a9, cortex-a12, cortex-a15, cortex-a17, krait, cortex-r4, cortex-r4f, cortex-r5, cortex-r7, cortex-r8, cortex-r52, sc300, cortex-m3, cortex-m4, cortex-m7, cortex-m23, cortex-m33, cortex-m35p, cortex-m55, cortex-m85, cortex-m52, cortex-a32, cortex-a35, cortex-a53, cortex-a55, cortex-a57, cortex-a72, cortex-a73, cortex-a75, cortex-a76, cortex-a76ae, cortex-a77, cortex-a78, cortex-a78c, cortex-a710, cortex-x1, cortex-x1c, neoverse-n1, neoverse-n2, neoverse-v1, cyclone, exynos-m3, exynos-m4, exynos-m5, kryo, iwmmxt, xscale, swift{{$}} // RUN: not %clang_cc1 -triple arm64--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix AARCH64 // AARCH64: error: unknown target CPU 'not-a-cpu' @@ -29,7 +29,7 @@ // RUN: not %clang_cc1 -triple nvptx--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix NVPTX // NVPTX: error: unknown target CPU 'not-a-cpu' -// NVPTX-NEXT: note: valid target CPU values are: sm_20, sm_21, sm_30, sm_32, sm_35, sm_37, sm_50, sm_52, sm_53, sm_60, sm_61, sm_62, sm_70, sm_72, sm_75, sm_80, sm_86, sm_87, sm_89, sm_90, gfx600, gfx601, gfx602, gfx700, gfx701, gfx702, gfx703, gfx704, gfx705, gfx801, gfx802, gfx803, gfx805, gfx810, gfx900, gfx902, gfx904, gfx906, gfx908, gfx909, gfx90a, gfx90c, gfx940, gfx941, gfx942, gfx1010, gfx1011, gfx1012, gfx1013, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034, gfx1035, gfx1036, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1200, gfx1201{{$}} +// NVPTX-NEXT: note: valid target CPU values are: sm_20, sm_21, sm_30, sm_32, sm_35, sm_37, sm_50, sm_52, sm_53, sm_60, sm_61, sm_62, sm_70, sm_72, sm_75, sm_80, sm_86, sm_87, sm_89, sm_90, sm_90a, gfx600, gfx601, gfx602, gfx700, gfx701, gfx702, gfx703, gfx704, gfx705, gfx801, gfx802, gfx803, gfx805, gfx810, gfx900, gfx902, gfx904, gfx906, gfx908, gfx909, gfx90a, gfx90c, gfx940, gfx941, gfx942, gfx1010, gfx1011, gfx1012, gfx1013, gfx1030, gfx1031, gfx1032, gfx1033, gfx1034, gfx1035, gfx1036, gfx1100, gfx1101, gfx1102, gfx1103, gfx1150, gfx1151, gfx1200, gfx1201{{$}} // RUN: not %clang_cc1 -triple r600--- -target-cpu not-a-cpu -fsyntax-only %s 2>&1 | FileCheck %s --check-prefix R600 // R600: error: unknown target CPU 'not-a-cpu' diff --git a/clang/test/Modules/decl-params-determinisim.m b/clang/test/Modules/decl-params-determinisim.m index 351403d9af94..9cf37ac4334c 100644 --- a/clang/test/Modules/decl-params-determinisim.m +++ b/clang/test/Modules/decl-params-determinisim.m @@ -28,23 +28,23 @@ // CHECK: vint64m1_t test_vsmul_vv_i64m1(vint64m1_t op1, vint64m1_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m1(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m1(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vsmul_vx_i64m1(vint64m1_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m1(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m1(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vsmul_vv_i64m2(vint64m2_t op1, vint64m2_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m2(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m2(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vsmul_vx_i64m2(vint64m2_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m2(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m2(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vsmul_vv_i64m4(vint64m4_t op1, vint64m4_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m4(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m4(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vsmul_vx_i64m4(vint64m4_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m4(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m4(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vsmul_vv_i64m8(vint64m8_t op1, vint64m8_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m8(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m8(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vsmul_vx_i64m8(vint64m8_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m8(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m8(op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vsmul_vv_i64m1_m(vbool64_t mask, vint64m1_t op1, vint64m1_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m1_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m1_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vsmul_vx_i64m1_m(vbool64_t mask, vint64m1_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m1_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m1_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vsmul_vv_i64m2_m(vbool32_t mask, vint64m2_t op1, vint64m2_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m2_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m2_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vsmul_vx_i64m2_m(vbool32_t mask, vint64m2_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m2_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m2_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vsmul_vv_i64m4_m(vbool16_t mask, vint64m4_t op1, vint64m4_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m4_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m4_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vsmul_vx_i64m4_m(vbool16_t mask, vint64m4_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m4_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m4_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vsmul_vv_i64m8_m(vbool8_t mask, vint64m8_t op1, vint64m8_t op2, size_t vl) { - return __riscv_vsmul_vv_i64m8_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vv_i64m8_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vsmul_vx_i64m8_m(vbool8_t mask, vint64m8_t op1, int64_t op2, size_t vl) { - return __riscv_vsmul_vx_i64m8_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vsmul_vx_i64m8_m(mask, op1, op2, __RISCV_VXRM_RNU, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulh_vv_i64m1(vint64m1_t op1, vint64m1_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m1(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m1(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulh_vx_i64m1(vint64m1_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m1(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m1(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulh_vv_i64m2(vint64m2_t op1, vint64m2_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m2(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m2(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulh_vx_i64m2(vint64m2_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m2(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m2(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulh_vv_i64m4(vint64m4_t op1, vint64m4_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m4(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m4(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulh_vx_i64m4(vint64m4_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m4(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m4(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulh_vv_i64m8(vint64m8_t op1, vint64m8_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m8(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m8(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulh_vx_i64m8(vint64m8_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m8(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m8(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulh_vv_i64m1_m(vbool64_t mask, vint64m1_t op1, vint64m1_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulh_vx_i64m1_m(vbool64_t mask, vint64m1_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulh_vv_i64m2_m(vbool32_t mask, vint64m2_t op1, vint64m2_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulh_vx_i64m2_m(vbool32_t mask, vint64m2_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulh_vv_i64m4_m(vbool16_t mask, vint64m4_t op1, vint64m4_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulh_vx_i64m4_m(vbool16_t mask, vint64m4_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulh_vv_i64m8_m(vbool8_t mask, vint64m8_t op1, vint64m8_t op2, size_t vl) { - return __riscv_vmulh_vv_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vv_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulh_vx_i64m8_m(vbool8_t mask, vint64m8_t op1, int64_t op2, size_t vl) { - return __riscv_vmulh_vx_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulh_vx_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m1_t test_vmulhu_vv_u64m1(vuint64m1_t op1, vuint64m1_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m1(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m1(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m1_t test_vmulhu_vx_u64m1(vuint64m1_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m1(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m1(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m2_t test_vmulhu_vv_u64m2(vuint64m2_t op1, vuint64m2_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m2(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m2(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m2_t test_vmulhu_vx_u64m2(vuint64m2_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m2(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m2(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m4_t test_vmulhu_vv_u64m4(vuint64m4_t op1, vuint64m4_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m4(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m4(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m4_t test_vmulhu_vx_u64m4(vuint64m4_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m4(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m4(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m8_t test_vmulhu_vv_u64m8(vuint64m8_t op1, vuint64m8_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m8(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m8(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m8_t test_vmulhu_vx_u64m8(vuint64m8_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m8(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m8(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m1_t test_vmulhu_vv_u64m1_m(vbool64_t mask, vuint64m1_t op1, vuint64m1_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m1_t test_vmulhu_vx_u64m1_m(vbool64_t mask, vuint64m1_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m2_t test_vmulhu_vv_u64m2_m(vbool32_t mask, vuint64m2_t op1, vuint64m2_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m2_t test_vmulhu_vx_u64m2_m(vbool32_t mask, vuint64m2_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m4_t test_vmulhu_vv_u64m4_m(vbool16_t mask, vuint64m4_t op1, vuint64m4_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m4_t test_vmulhu_vx_u64m4_m(vbool16_t mask, vuint64m4_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m8_t test_vmulhu_vv_u64m8_m(vbool8_t mask, vuint64m8_t op1, vuint64m8_t op2, size_t vl) { - return __riscv_vmulhu_vv_u64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vv_u64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vuint64m8_t test_vmulhu_vx_u64m8_m(vbool8_t mask, vuint64m8_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhu_vx_u64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhu_vx_u64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulhsu_vv_i64m1(vint64m1_t op1, vuint64m1_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m1(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m1(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulhsu_vx_i64m1(vint64m1_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m1(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m1(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulhsu_vv_i64m2(vint64m2_t op1, vuint64m2_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m2(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m2(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulhsu_vx_i64m2(vint64m2_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m2(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m2(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulhsu_vv_i64m4(vint64m4_t op1, vuint64m4_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m4(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m4(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulhsu_vx_i64m4(vint64m4_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m4(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m4(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulhsu_vv_i64m8(vint64m8_t op1, vuint64m8_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m8(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m8(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulhsu_vx_i64m8(vint64m8_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m8(op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m8(op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulhsu_vv_i64m1_m(vbool64_t mask, vint64m1_t op1, vuint64m1_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m1_t test_vmulhsu_vx_i64m1_m(vbool64_t mask, vint64m1_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m1_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulhsu_vv_i64m2_m(vbool32_t mask, vint64m2_t op1, vuint64m2_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m2_t test_vmulhsu_vx_i64m2_m(vbool32_t mask, vint64m2_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m2_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulhsu_vv_i64m4_m(vbool16_t mask, vint64m4_t op1, vuint64m4_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m4_t test_vmulhsu_vx_i64m4_m(vbool16_t mask, vint64m4_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m4_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulhsu_vv_i64m8_m(vbool8_t mask, vint64m8_t op1, vuint64m8_t op2, size_t vl) { - return __riscv_vmulhsu_vv_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vv_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } vint64m8_t test_vmulhsu_vx_i64m8_m(vbool8_t mask, vint64m8_t op1, uint64_t op2, size_t vl) { - return __riscv_vmulhsu_vx_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires: v}} */ + return __riscv_vmulhsu_vx_i64m8_m(mask, op1, op2, vl); /* expected-error {{builtin requires at least one of the following extensions: v}} */ } diff --git a/clang/test/Sema/scoped-atomic-ops.c b/clang/test/Sema/scoped-atomic-ops.c new file mode 100644 index 000000000000..59e638c64666 --- /dev/null +++ b/clang/test/Sema/scoped-atomic-ops.c @@ -0,0 +1,101 @@ +// RUN: %clang_cc1 -x c -triple=amdgcn-amd-amdhsa -verify -fsyntax-only %s +// RUN: %clang_cc1 -x c -triple=x86_64-pc-linux-gnu -verify -fsyntax-only %s + +int fi1a(int *i) { + int v; + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + __scoped_atomic_load(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return v; +} + +int fi1b(int *i) { + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 3, have 2}} + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *i = __scoped_atomic_load_n(i, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return *i; +} + +int fi2a(int *i) { + int v; + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + __scoped_atomic_store(i, &v, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return v; +} + +void fi2b(int *i) { + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + __scoped_atomic_store_n(i, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); +} + +void fi3a(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); +} + +void fi3b(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *b = __scoped_atomic_fetch_sub(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *c = __scoped_atomic_fetch_and(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *d = __scoped_atomic_fetch_or(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *e = __scoped_atomic_fetch_xor(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *f = __scoped_atomic_fetch_nand(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *g = __scoped_atomic_fetch_min(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} + *h = __scoped_atomic_fetch_max(1, 1, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); // expected-error {{address argument to atomic builtin must be a pointer ('int' invalid)}} +} + +void fi3c(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED); // expected-error {{too few arguments to function call, expected 4, have 3}} +} + +void fi3d(int *a, int *b, int *c, int *d, int *e, int *f, int *g, int *h) { + *a = __scoped_atomic_fetch_add(a, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *b = __scoped_atomic_fetch_sub(b, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *c = __scoped_atomic_fetch_and(c, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *d = __scoped_atomic_fetch_or(d, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *e = __scoped_atomic_fetch_xor(e, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *f = __scoped_atomic_fetch_nand(f, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *g = __scoped_atomic_fetch_min(g, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} + *h = __scoped_atomic_fetch_max(h, 1, __ATOMIC_RELAXED, 42); // expected-error {{synchronization scope argument to atomic operation is invalid}} +} + +int fi4a(int *i) { + int cmp = 0; + int desired = 1; + return __scoped_atomic_compare_exchange(i, &cmp, &desired, 0, + __ATOMIC_ACQUIRE, __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SYSTEM); +} + +int fi5a(int *i) { + int cmp = 0; + return __scoped_atomic_compare_exchange_n(i, &cmp, 1, 1, __ATOMIC_ACQUIRE, + __ATOMIC_ACQUIRE, + __MEMORY_SCOPE_SYSTEM); +} + +int fi6a(int *c, int *d) { + int ret; + __scoped_atomic_exchange(c, d, &ret, __ATOMIC_RELAXED, __MEMORY_SCOPE_SYSTEM); + return ret; +} + +int fi7a(_Bool *c) { + return __scoped_atomic_exchange_n(c, 1, __ATOMIC_RELAXED, + __MEMORY_SCOPE_SYSTEM); +} diff --git a/clang/test/Sema/switch-default.c b/clang/test/Sema/switch-default.c new file mode 100644 index 000000000000..854b561b37c4 --- /dev/null +++ b/clang/test/Sema/switch-default.c @@ -0,0 +1,17 @@ +// RUN: %clang_cc1 -fsyntax-only -verify -Wswitch-default %s + +int f1(int a) { + switch (a) { // expected-warning {{'switch' missing 'default' label}} + case 1: a++; break; + case 2: a += 2; break; + } + return a; +} + +int f2(int a) { + switch (a) { // no-warning + default: + ; + } + return a; +} diff --git a/clang/test/SemaCXX/coro-lifetimebound.cpp b/clang/test/SemaCXX/coro-lifetimebound.cpp index d3e2d673ebb3..b4dc029a1398 100644 --- a/clang/test/SemaCXX/coro-lifetimebound.cpp +++ b/clang/test/SemaCXX/coro-lifetimebound.cpp @@ -1,4 +1,4 @@ -// RUN: %clang_cc1 -triple x86_64-apple-darwin9 %s -std=c++20 -fsyntax-only -verify -Wall -Wextra -Wno-error=unreachable-code -Wno-unused +// RUN: %clang_cc1 -triple x86_64-apple-darwin9 %s -std=c++20 -fsyntax-only -verify -Wall -Wextra -Wno-error=unreachable-code -Wno-unused -Wno-c++23-lambda-attributes #include "Inputs/std-coroutine.h" @@ -64,14 +64,8 @@ Co bar_coro(const int &b, int c) { : bar_coro(0, 1); // expected-warning {{returning address of local temporary object}} } -#define CORO_WRAPPER \ - _Pragma("clang diagnostic push") \ - _Pragma("clang diagnostic ignored \"-Wc++23-extensions\"") \ - [[clang::coro_wrapper]] \ - _Pragma("clang diagnostic pop") - void lambdas() { - auto unsafe_lambda = [] CORO_WRAPPER (int b) { + auto unsafe_lambda = [] [[clang::coro_wrapper]] (int b) { return foo_coro(b); // expected-warning {{address of stack memory associated with parameter}} }; auto coro_lambda = [] (const int&) -> Co { diff --git a/clang/test/SemaCXX/coro-return-type-and-wrapper.cpp b/clang/test/SemaCXX/coro-return-type-and-wrapper.cpp index 5f8076f1c782..ac49e03ba9d9 100644 --- a/clang/test/SemaCXX/coro-return-type-and-wrapper.cpp +++ b/clang/test/SemaCXX/coro-return-type-and-wrapper.cpp @@ -1,4 +1,4 @@ -// RUN: %clang_cc1 -triple x86_64-apple-darwin9 %s -std=c++20 -fsyntax-only -verify -Wall -Wextra +// RUN: %clang_cc1 -triple x86_64-apple-darwin9 %s -std=c++20 -fsyntax-only -verify -Wall -Wextra -Wno-c++23-lambda-attributes #include "Inputs/std-coroutine.h" using std::suspend_always; @@ -45,11 +45,6 @@ Co non_marked_wrapper(int b) { return foo_coro(b); } } // namespace using_decl namespace lambdas { -#define CORO_WRAPPER \ - _Pragma("clang diagnostic push") \ - _Pragma("clang diagnostic ignored \"-Wc++23-extensions\"") \ - [[clang::coro_wrapper]] \ - _Pragma("clang diagnostic pop") void foo() { auto coro_lambda = []() -> Gen { @@ -59,7 +54,7 @@ void foo() { auto not_allowed_wrapper = []() -> Gen { return foo_coro(1); }; - auto allowed_wrapper = [] CORO_WRAPPER() -> Gen { + auto allowed_wrapper = [] [[clang::coro_wrapper]] () -> Gen { return foo_coro(1); }; } diff --git a/clang/test/SemaCXX/cxx1z-copy-omission.cpp b/clang/test/SemaCXX/cxx1z-copy-omission.cpp index a850cf6143cd..f46a17af8338 100644 --- a/clang/test/SemaCXX/cxx1z-copy-omission.cpp +++ b/clang/test/SemaCXX/cxx1z-copy-omission.cpp @@ -171,3 +171,30 @@ namespace CtorTemplateBeatsNonTemplateConversionFn { Foo f(Derived d) { return d; } // expected-error {{invokes a deleted function}} Foo g(Derived d) { return Foo(d); } // ok, calls constructor } + +// Make sure we don't consider conversion functions for guaranteed copy elision +namespace GH39319 { +struct A { + A(); + A(const A&) = delete; // expected-note {{'A' has been explicitly marked deleted here}} +}; +struct B { + operator A(); +} C; +A::A() : A(C) {} // expected-error {{call to deleted constructor of}} + +struct A2 { + struct B2 { + operator A2(); + }; + A2() : A2(B2()) {} // expected-error {{call to deleted constructor of}} + A2(const A2&) = delete; // expected-note {{'A2' has been explicitly marked deleted here}} +}; + +template +class B3 : A3 { + template()> // expected-warning 2{{use of function template name with no prior declaration in function call with explicit}} + B3(); +}; B3(); // expected-error {{deduction guide declaration without trailing return type}} \ + // expected-note {{while building implicit deduction guide first needed here}} +} diff --git a/clang/test/SemaCXX/friend.cpp b/clang/test/SemaCXX/friend.cpp index 367d6a6c1807..53e6bbfcf42a 100644 --- a/clang/test/SemaCXX/friend.cpp +++ b/clang/test/SemaCXX/friend.cpp @@ -162,7 +162,7 @@ namespace test9 { class C { }; struct A { - friend void C::f(int, int, int) {} // expected-error {{friend function definition cannot be qualified with 'C::'}} + friend void C::f(int, int, int) {} // expected-error {{friend declaration of 'f' does not match any declaration in 'test9::C'}} }; } diff --git a/clang/test/SemaCXX/ms-constexpr-invalid.cpp b/clang/test/SemaCXX/ms-constexpr-invalid.cpp new file mode 100644 index 000000000000..e5bec0c7119b --- /dev/null +++ b/clang/test/SemaCXX/ms-constexpr-invalid.cpp @@ -0,0 +1,52 @@ +// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.33 -std=c++20 -verify %s +// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.33 -std=c++17 -verify %s + +// Check explicitly invalid code + +void runtime() {} // expected-note {{declared here}} + +[[msvc::constexpr]] void f0() { runtime(); } // expected-error {{constexpr function never produces a constant expression}} \ + // expected-note {{non-constexpr function 'runtime' cannot be used in a constant expression}} +[[msvc::constexpr]] constexpr void f1() {} // expected-error {{attribute 'msvc::constexpr' cannot be applied to the constexpr function 'f1'}} +#if __cplusplus >= 202202L +[[msvc::constexpr]] consteval void f2() {} // expected-error {{attribute 'msvc::constexpr' cannot be applied to the consteval function 'f1'}} +#endif + +struct B1 {}; +struct D1 : virtual B1 { // expected-note {{virtual base class declared here}} + [[msvc::constexpr]] D1() {} // expected-error {{constexpr constructor not allowed in struct with virtual base class}} +}; + +struct [[msvc::constexpr]] S2{}; // expected-error {{'constexpr' attribute only applies to functions and return statements}} + +// Check invalid code mixed with valid code + +[[msvc::constexpr]] int f4(int x) { return x > 1 ? 1 + f4(x / 2) : 0; } // expected-note {{non-constexpr function 'f4' cannot be used in a constant expression}} \ + // expected-note {{declared here}} \ + // expected-note {{declared here}} \ + // expected-note {{declared here}} +constexpr bool f5() { [[msvc::constexpr]] return f4(32) == 5; } // expected-note {{in call to 'f4(32)'}} +static_assert(f5()); // expected-error {{static assertion expression is not an integral constant expression}} \ + // expected-note {{in call to 'f5()'}} + +int f6(int x) { [[msvc::constexpr]] return x > 1 ? 1 + f6(x / 2) : 0; } // expected-note {{declared here}} \ + // expected-note {{declared here}} +constexpr bool f7() { [[msvc::constexpr]] return f6(32) == 5; } // expected-error {{constexpr function never produces a constant expression}} \ + // expected-note {{non-constexpr function 'f6' cannot be used in a constant expression}} \ + // expected-note {{non-constexpr function 'f6' cannot be used in a constant expression}} +static_assert(f7()); // expected-error {{static assertion expression is not an integral constant expression}} \ + // expected-note {{in call to 'f7()'}} + +constexpr bool f8() { // expected-error {{constexpr function never produces a constant expression}} + [[msvc::constexpr]] f4(32); // expected-error {{'constexpr' attribute only applies to functions and return statements}} \ + // expected-note {{non-constexpr function 'f4' cannot be used in a constant expression}} \ + // expected-note {{non-constexpr function 'f4' cannot be used in a constant expression}} + [[msvc::constexpr]] int i5 = f4(32); // expected-error {{'constexpr' attribute only applies to functions and return statements}} + return i5 == 5; +} +static_assert(f8()); // expected-error {{static assertion expression is not an integral constant expression}} \ + // expected-note {{in call to 'f8()'}} + +#if __cplusplus == 201702L +struct S1 { [[msvc::constexpr]] virtual bool vm() const { return true; } }; // expected-error {{attribute 'msvc::constexpr' ignored, it only applies to function definitions and return statements}} +#endif diff --git a/clang/test/SemaCXX/ms-constexpr-new.cpp b/clang/test/SemaCXX/ms-constexpr-new.cpp new file mode 100644 index 000000000000..30567740b2ec --- /dev/null +++ b/clang/test/SemaCXX/ms-constexpr-new.cpp @@ -0,0 +1,16 @@ +// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.33 -std=c++20 -verify=supported %s +// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.32 -std=c++20 -verify=unsupported %s +// supported-no-diagnostics + +[[nodiscard]] +[[msvc::constexpr]] // unsupported-warning {{unknown attribute 'constexpr' ignored}} +inline void* operator new(decltype(sizeof(void*)), void* p) noexcept { return p; } + +namespace std { + constexpr int* construct_at(int* p, int v) { + [[msvc::constexpr]] return ::new (p) int(v); // unsupported-warning {{unknown attribute 'constexpr' ignored}} + } +} + +constexpr bool check_construct_at() { int x; return *std::construct_at(&x, 42) == 42; } +static_assert(check_construct_at()); diff --git a/clang/test/SemaCXX/ms-constexpr.cpp b/clang/test/SemaCXX/ms-constexpr.cpp new file mode 100644 index 000000000000..79f71a34cb7d --- /dev/null +++ b/clang/test/SemaCXX/ms-constexpr.cpp @@ -0,0 +1,37 @@ +// RUN: %clang_cc1 -fms-compatibility -fms-compatibility-version=19.33 -std=c++20 -verify %s + +[[msvc::constexpr]] int log2(int x) { [[msvc::constexpr]] return x > 1 ? 1 + log2(x / 2) : 0; } +constexpr bool test_log2() { [[msvc::constexpr]] return log2(32) == 5; } +static_assert(test_log2()); + +[[msvc::constexpr]] int get_value(int x) +{ + switch (x) + { + case 42: return 1337; + default: + if (x < 0) [[msvc::constexpr]] return log2(-x); + else return x; + } +} + +constexpr bool test_complex_expr() { + [[msvc::constexpr]] return get_value(get_value(42) - 1337 + get_value(-32) - 5 + (get_value(1) ? get_value(0) : get_value(2))) == get_value(0); +} +static_assert(test_complex_expr()); + +constexpr bool get_constexpr_true() { return true; } +[[msvc::constexpr]] bool get_msconstexpr_true() { return get_constexpr_true(); } +constexpr bool test_get_msconstexpr_true() { [[msvc::constexpr]] return get_msconstexpr_true(); } +static_assert(test_get_msconstexpr_true()); + +// TODO (#72149): Add support for [[msvc::constexpr]] constructor; this code is valid for MSVC. +struct S2 { + [[msvc::constexpr]] S2() {} + [[msvc::constexpr]] bool value() { return true; } + static constexpr bool check() { [[msvc::constexpr]] return S2{}.value(); } // expected-error {{constexpr function never produces a constant expression}} \ + // expected-note {{non-literal type 'S2' cannot be used in a constant expression}} \ + // expected-note {{non-literal type 'S2' cannot be used in a constant expression}} +}; +static_assert(S2::check()); // expected-error {{static assertion expression is not an integral constant expression}} \ + // expected-note {{in call to 'check()'}} diff --git a/clang/test/SemaCXX/warn-thread-safety-analysis.cpp b/clang/test/SemaCXX/warn-thread-safety-analysis.cpp index 205cfa284f6c..dfb966d3b590 100644 --- a/clang/test/SemaCXX/warn-thread-safety-analysis.cpp +++ b/clang/test/SemaCXX/warn-thread-safety-analysis.cpp @@ -1702,6 +1702,8 @@ struct TestScopedLockable { bool getBool(); + bool lock2Bool(MutexLock); + void foo1() { MutexLock mulock(&mu1); a = 5; @@ -1718,6 +1720,12 @@ struct TestScopedLockable { MutexLock{&mu1}, a = 5; } + void temporary_cfg(int x) { + // test the case where a pair of temporary Ctor and Dtor is in different CFG blocks + lock2Bool(MutexLock{&mu1}) || x; + MutexLock{&mu1}; // no-warn + } + void lifetime_extension() { const MutexLock &mulock = MutexLock(&mu1); a = 5; diff --git a/clang/test/SemaCXX/warn-unsafe-buffer-usage-fixits-add-assign.cpp b/clang/test/SemaCXX/warn-unsafe-buffer-usage-fixits-add-assign.cpp new file mode 100644 index 000000000000..5c03cc10025c --- /dev/null +++ b/clang/test/SemaCXX/warn-unsafe-buffer-usage-fixits-add-assign.cpp @@ -0,0 +1,59 @@ +// RUN: %clang_cc1 -std=c++20 -Wunsafe-buffer-usage \ +// RUN: -fsafe-buffer-usage-suggestions \ +// RUN: -fdiagnostics-parseable-fixits %s 2>&1 | FileCheck %s +void foo(int * , int *); + +void add_assign_test(unsigned int n, int *a, int y) { + int *p = new int[10]; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:11}:"std::span p" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:12-[[@LINE-2]]:12}:"{" + // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:23-[[@LINE-3]]:23}:", 10}" + p += 2; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:7}:"p = p.subspan(" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:9-[[@LINE-2]]:9}:")" + + int *r = p; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:11}:"std::span r" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:12-[[@LINE-2]]:12}:"{" + // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:13-[[@LINE-3]]:13}:", <# placeholder #>}" + while (*r != 0) { + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:10-[[@LINE-1]]:11}:"" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:12-[[@LINE-2]]:12}:"[0]" + r += 2; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:5-[[@LINE-1]]:9}:"r = r.subspan(" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:11-[[@LINE-2]]:11}:")" + } + + if (*p == 0) { + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:7-[[@LINE-1]]:8}:"" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:9-[[@LINE-2]]:9}:"[0]" + p += n; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:5-[[@LINE-1]]:9}:"p = p.subspan(" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:11-[[@LINE-2]]:11}:")" + } + + if (*p == 1) + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:7-[[@LINE-1]]:8}:"" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:9-[[@LINE-2]]:9}:"[0]" + p += 3; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:5-[[@LINE-1]]:9}:"p = p.subspan(" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:11-[[@LINE-2]]:11}:")" + + a += -9; + // CHECK-NOT: fix-it:"{{.*}}":{[[@LINE-1]]:5-[[@LINE-1]]:9}:"p = p.subspan(" + + a += y; + // CHECK-NOT: fix-it:"{{.*}}":{[[@LINE-1]]:5-[[@LINE-1]]:9}:"p = p.subspan(" +} + +int expr_test(unsigned x, int *q, int y) { + char *p = new char[8]; + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:12}:"std::span p" + // CHECK: fix-it:"{{.*}}":{[[@LINE-2]]:13-[[@LINE-2]]:13}:"{" + // CHECK: fix-it:"{{.*}}":{[[@LINE-3]]:24-[[@LINE-3]]:24}:", 8}" + p += (x + 1); + // CHECK: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:7}:"p = p.subspan" + + q += (y + 7); + // CHECK-NOT: fix-it:"{{.*}}":{[[@LINE-1]]:3-[[@LINE-1]]:7}:"q = q.subspan" +} diff --git a/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl b/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl index a9f6567aa00b..7e79ae3bf005 100644 --- a/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/RWBuffers.hlsl @@ -5,6 +5,14 @@ typedef vector float3; RWBuffer Buffer; +// expected-error@+2 {{class template 'RWBuffer' requires template arguments}} +// expected-note@*:* {{template declaration from hidden source: template class RWBuffer final}} +RWBuffer BufferErr1; + +// expected-error@+2 {{too few template arguments for class template 'RWBuffer'}} +// expected-note@*:* {{template declaration from hidden source: template class RWBuffer final}} +RWBuffer<> BufferErr2; + [numthreads(1,1,1)] void main() { (void)Buffer.h; // expected-error {{'h' is a private member of 'hlsl::RWBuffer'}} diff --git a/clang/test/SemaHLSL/BuiltIns/vector-errors.hlsl b/clang/test/SemaHLSL/BuiltIns/vector-errors.hlsl index d2598a57400b..6aedb9304ed9 100644 --- a/clang/test/SemaHLSL/BuiltIns/vector-errors.hlsl +++ b/clang/test/SemaHLSL/BuiltIns/vector-errors.hlsl @@ -2,17 +2,16 @@ // Some bad declarations hlsl::vector ShouldWorkSomeday; // expected-error{{use of alias template 'hlsl::vector' requires template arguments}} +// expected-note@*:* {{template declaration from hidden source: template using vector = element __attribute__((ext_vector_type(element_count)))}} hlsl::vector<1> BadVec; // expected-error{{template argument for template type parameter must be a type}} -// expected-note@*:* {{template is declared here}} -// expected-note@*:* {{template parameter is declared here}} - +// expected-note@*:* {{template parameter from hidden source: class element = float}} hlsl::vector AnotherBadVec; // expected-error{{template argument for non-type template parameter must be an expression}} -// expected-note@*:* {{template parameter is declared here}} +// expected-note@*:* {{template parameter from hidden source: int element_count = 4}} hlsl::vector YABV; // expected-error{{too many template arguments for alias template 'vector'}} -// expected-note@*:* {{template is declared here}} +// expected-note@*:* {{template declaration from hidden source: template using vector = element __attribute__((ext_vector_type(element_count)))}} // This code is rejected by clang because clang puts the HLSL built-in types // into the HLSL namespace. diff --git a/clang/test/SemaTemplate/GH71595.cpp b/clang/test/SemaTemplate/GH71595.cpp new file mode 100644 index 000000000000..7d34d1bf054e --- /dev/null +++ b/clang/test/SemaTemplate/GH71595.cpp @@ -0,0 +1,34 @@ +// RUN: %clang_cc1 -std=c++20 -verify %s + +template +concept C = true; + +class non_temp { + template T> + friend void f(); + + non_temp(); +}; + +template T> +void f() { + auto v = non_temp(); +} + +template +class temp { + template T> + friend void g(); + + temp(); // expected-note {{implicitly declared private here}} +}; + +template> T> +void g() { + auto v = temp(); // expected-error {{calling a private constructor of class 'temp'}} +} + +void h() { + f(); + g(); // expected-note {{in instantiation of function template specialization 'g' requested here}} +} diff --git a/clang/tools/clang-format/ClangFormat.cpp b/clang/tools/clang-format/ClangFormat.cpp index 829f85b93bc7..d2e3d8d43aef 100644 --- a/clang/tools/clang-format/ClangFormat.cpp +++ b/clang/tools/clang-format/ClangFormat.cpp @@ -398,8 +398,8 @@ class ClangFormatDiagConsumer : public DiagnosticConsumer { }; // Returns true on error. -static bool format(StringRef FileName) { - if (!OutputXML && Inplace && FileName == "-") { +static bool format(StringRef FileName, bool IsSTDIN) { + if (!OutputXML && Inplace && IsSTDIN) { errs() << "error: cannot use -i when reading from stdin.\n"; return false; } @@ -423,7 +423,7 @@ static bool format(StringRef FileName) { if (InvalidBOM) { errs() << "error: encoding with unsupported byte order mark \"" << InvalidBOM << "\" detected"; - if (FileName != "-") + if (!IsSTDIN) errs() << " in file '" << FileName << "'"; errs() << ".\n"; return true; @@ -432,7 +432,7 @@ static bool format(StringRef FileName) { std::vector Ranges; if (fillRanges(Code.get(), Ranges)) return true; - StringRef AssumedFileName = (FileName == "-") ? AssumeFileName : FileName; + StringRef AssumedFileName = IsSTDIN ? AssumeFileName : FileName; if (AssumedFileName.empty()) { llvm::errs() << "error: empty filenames are not allowed\n"; return true; @@ -544,28 +544,23 @@ static void PrintVersion(raw_ostream &OS) { } // Dump the configuration. -static int dumpConfig() { - StringRef FileName; +static int dumpConfig(bool IsSTDIN) { std::unique_ptr Code; - if (FileNames.empty()) { - // We can't read the code to detect the language if there's no - // file name, so leave Code empty here. - FileName = AssumeFileName; - } else { - // Read in the code in case the filename alone isn't enough to - // detect the language. + // We can't read the code to detect the language if there's no file name. + if (!IsSTDIN) { + // Read in the code in case the filename alone isn't enough to detect the + // language. ErrorOr> CodeOrErr = MemoryBuffer::getFileOrSTDIN(FileNames[0]); if (std::error_code EC = CodeOrErr.getError()) { llvm::errs() << EC.message() << "\n"; return 1; } - FileName = (FileNames[0] == "-") ? AssumeFileName : FileNames[0]; Code = std::move(CodeOrErr.get()); } llvm::Expected FormatStyle = - clang::format::getStyle(Style, FileName, FallbackStyle, - Code ? Code->getBuffer() : ""); + clang::format::getStyle(Style, IsSTDIN ? AssumeFileName : FileNames[0], + FallbackStyle, Code ? Code->getBuffer() : ""); if (!FormatStyle) { llvm::errs() << llvm::toString(FormatStyle.takeError()) << "\n"; return 1; @@ -596,8 +591,11 @@ int main(int argc, const char **argv) { return 0; } + if (FileNames.empty()) + FileNames.push_back("-"); + if (DumpConfig) - return dumpConfig(); + return dumpConfig(FileNames[0] == "-"); if (!Files.empty()) { std::ifstream ExternalFileOfFiles{std::string(Files)}; @@ -610,11 +608,6 @@ int main(int argc, const char **argv) { errs() << "Clang-formating " << LineNo << " files\n"; } - bool Error = false; - if (FileNames.empty()) { - Error = clang::format::format("-"); - return Error ? 1 : 0; - } if (FileNames.size() != 1 && (!Offsets.empty() || !Lengths.empty() || !LineRanges.empty())) { errs() << "error: -offset, -length and -lines can only be used for " @@ -623,12 +616,13 @@ int main(int argc, const char **argv) { } unsigned FileNo = 1; + bool Error = false; for (const auto &FileName : FileNames) { if (Verbose) { errs() << "Formatting [" << FileNo++ << "/" << FileNames.size() << "] " << FileName << "\n"; } - Error |= clang::format::format(FileName); + Error |= clang::format::format(FileName, FileName == "-"); } return Error ? 1 : 0; } diff --git a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp index db0ce3e2a190..5d2fe98fe560 100644 --- a/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp +++ b/clang/tools/clang-linker-wrapper/ClangLinkerWrapper.cpp @@ -428,7 +428,7 @@ Expected clang(ArrayRef InputFiles, const ArgList &Args) { std::back_inserter(CmdArgs)); for (StringRef Arg : Args.getAllArgValues(OPT_linker_arg_EQ)) - CmdArgs.push_back(Args.MakeArgString("-Wl," + Arg)); + CmdArgs.push_back(Args.MakeArgString(Arg)); for (StringRef Arg : Args.getAllArgValues(OPT_builtin_bitcode_EQ)) { if (llvm::Triple(Arg.split('=').first) == Triple) diff --git a/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp b/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp index 3e1dd874216c..f4f500b17357 100644 --- a/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp +++ b/clang/tools/clang-linker-wrapper/OffloadWrapper.cpp @@ -26,19 +26,6 @@ namespace { constexpr unsigned CudaFatMagic = 0x466243b1; constexpr unsigned HIPFatMagic = 0x48495046; -/// Copied from clang/CGCudaRuntime.h. -enum OffloadEntryKindFlag : uint32_t { - /// Mark the entry as a global entry. This indicates the presense of a - /// kernel if the size size field is zero and a variable otherwise. - OffloadGlobalEntry = 0x0, - /// Mark the entry as a managed global variable. - OffloadGlobalManagedEntry = 0x1, - /// Mark the entry as a surface variable. - OffloadGlobalSurfaceEntry = 0x2, - /// Mark the entry as a texture variable. - OffloadGlobalTextureEntry = 0x3, -}; - IntegerType *getSizeTTy(Module &M) { return M.getDataLayout().getIntPtrType(M.getContext()); } @@ -333,6 +320,24 @@ Function *createRegisterGlobalsFunction(Module &M, bool IsHIP) { FunctionCallee RegVar = M.getOrInsertFunction( IsHIP ? "__hipRegisterVar" : "__cudaRegisterVar", RegVarTy); + // Get the __cudaRegisterSurface function declaration. + auto *RegSurfaceTy = + FunctionType::get(Type::getVoidTy(C), + {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, + Type::getInt32Ty(C), Type::getInt32Ty(C)}, + /*isVarArg=*/false); + FunctionCallee RegSurface = M.getOrInsertFunction( + IsHIP ? "__hipRegisterSurface" : "__cudaRegisterSurface", RegSurfaceTy); + + // Get the __cudaRegisterTexture function declaration. + auto *RegTextureTy = FunctionType::get( + Type::getVoidTy(C), + {Int8PtrPtrTy, Int8PtrTy, Int8PtrTy, Int8PtrTy, Type::getInt32Ty(C), + Type::getInt32Ty(C), Type::getInt32Ty(C)}, + /*isVarArg=*/false); + FunctionCallee RegTexture = M.getOrInsertFunction( + IsHIP ? "__hipRegisterTexture" : "__cudaRegisterTexture", RegTextureTy); + auto *RegGlobalsTy = FunctionType::get(Type::getVoidTy(C), Int8PtrPtrTy, /*isVarArg*/ false); auto *RegGlobalsFn = @@ -375,7 +380,31 @@ Function *createRegisterGlobalsFunction(Module &M, bool IsHIP) { Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry, {ConstantInt::get(getSizeTTy(M), 0), ConstantInt::get(Type::getInt32Ty(C), 3)}); - auto *Flags = Builder.CreateLoad(Type::getInt32Ty(C), FlagsPtr, "flag"); + auto *Flags = Builder.CreateLoad(Type::getInt32Ty(C), FlagsPtr, "flags"); + auto *DataPtr = + Builder.CreateInBoundsGEP(offloading::getEntryTy(M), Entry, + {ConstantInt::get(getSizeTTy(M), 0), + ConstantInt::get(Type::getInt32Ty(C), 4)}); + auto *Data = Builder.CreateLoad(Type::getInt32Ty(C), DataPtr, "textype"); + auto *Kind = Builder.CreateAnd( + Flags, ConstantInt::get(Type::getInt32Ty(C), 0x7), "type"); + + // Extract the flags stored in the bit-field and convert them to C booleans. + auto *ExternBit = Builder.CreateAnd( + Flags, ConstantInt::get(Type::getInt32Ty(C), + llvm::offloading::OffloadGlobalExtern)); + auto *Extern = Builder.CreateLShr( + ExternBit, ConstantInt::get(Type::getInt32Ty(C), 3), "extern"); + auto *ConstantBit = Builder.CreateAnd( + Flags, ConstantInt::get(Type::getInt32Ty(C), + llvm::offloading::OffloadGlobalConstant)); + auto *Const = Builder.CreateLShr( + ConstantBit, ConstantInt::get(Type::getInt32Ty(C), 4), "constant"); + auto *NormalizedBit = Builder.CreateAnd( + Flags, ConstantInt::get(Type::getInt32Ty(C), + llvm::offloading::OffloadGlobalNormalized)); + auto *Normalized = Builder.CreateLShr( + NormalizedBit, ConstantInt::get(Type::getInt32Ty(C), 5), "normalized"); auto *FnCond = Builder.CreateICmpEQ(Size, ConstantInt::getNullValue(getSizeTTy(M))); Builder.CreateCondBr(FnCond, IfThenBB, IfElseBB); @@ -392,30 +421,37 @@ Function *createRegisterGlobalsFunction(Module &M, bool IsHIP) { Builder.CreateBr(IfEndBB); Builder.SetInsertPoint(IfElseBB); - auto *Switch = Builder.CreateSwitch(Flags, IfEndBB); + auto *Switch = Builder.CreateSwitch(Kind, IfEndBB); // Create global variable registration code. Builder.SetInsertPoint(SwGlobalBB); - Builder.CreateCall(RegVar, {RegGlobalsFn->arg_begin(), Addr, Name, Name, - ConstantInt::get(Type::getInt32Ty(C), 0), Size, - ConstantInt::get(Type::getInt32Ty(C), 0), - ConstantInt::get(Type::getInt32Ty(C), 0)}); + Builder.CreateCall(RegVar, + {RegGlobalsFn->arg_begin(), Addr, Name, Name, Extern, Size, + Const, ConstantInt::get(Type::getInt32Ty(C), 0)}); Builder.CreateBr(IfEndBB); - Switch->addCase(Builder.getInt32(OffloadGlobalEntry), SwGlobalBB); + Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalEntry), + SwGlobalBB); // Create managed variable registration code. Builder.SetInsertPoint(SwManagedBB); Builder.CreateBr(IfEndBB); - Switch->addCase(Builder.getInt32(OffloadGlobalManagedEntry), SwManagedBB); + Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalManagedEntry), + SwManagedBB); // Create surface variable registration code. Builder.SetInsertPoint(SwSurfaceBB); + Builder.CreateCall( + RegSurface, {RegGlobalsFn->arg_begin(), Addr, Name, Name, Data, Extern}); Builder.CreateBr(IfEndBB); - Switch->addCase(Builder.getInt32(OffloadGlobalSurfaceEntry), SwSurfaceBB); + Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalSurfaceEntry), + SwSurfaceBB); // Create texture variable registration code. Builder.SetInsertPoint(SwTextureBB); + Builder.CreateCall(RegTexture, {RegGlobalsFn->arg_begin(), Addr, Name, Name, + Data, Normalized, Extern}); Builder.CreateBr(IfEndBB); - Switch->addCase(Builder.getInt32(OffloadGlobalTextureEntry), SwTextureBB); + Switch->addCase(Builder.getInt32(llvm::offloading::OffloadGlobalTextureEntry), + SwTextureBB); Builder.SetInsertPoint(IfEndBB); auto *NewEntry = Builder.CreateInBoundsGEP( diff --git a/clang/tools/libclang/CXIndexDataConsumer.cpp b/clang/tools/libclang/CXIndexDataConsumer.cpp index 5ca484fbc8cd..c1022263a512 100644 --- a/clang/tools/libclang/CXIndexDataConsumer.cpp +++ b/clang/tools/libclang/CXIndexDataConsumer.cpp @@ -1074,8 +1074,8 @@ CXIndexDataConsumer::getClientContainerForDC(const DeclContext *DC) const { return DC ? ContainerMap.lookup(DC) : nullptr; } -CXIdxClientFile CXIndexDataConsumer::getIndexFile(const FileEntry *File) { - return File ? FileMap.lookup(File) : nullptr; +CXIdxClientFile CXIndexDataConsumer::getIndexFile(OptionalFileEntryRef File) { + return File ? FileMap.lookup(*File) : nullptr; } CXIdxLoc CXIndexDataConsumer::getIndexLoc(SourceLocation Loc) const { @@ -1104,8 +1104,8 @@ void CXIndexDataConsumer::translateLoc(SourceLocation Loc, if (FID.isInvalid()) return; - - OptionalFileEntryRefDegradesToFileEntryPtr FE = SM.getFileEntryRefForID(FID); + + OptionalFileEntryRef FE = SM.getFileEntryRefForID(FID); if (indexFile) *indexFile = getIndexFile(FE); if (file) diff --git a/clang/tools/libclang/CXIndexDataConsumer.h b/clang/tools/libclang/CXIndexDataConsumer.h index afa2239ed653..54a3add3a9c8 100644 --- a/clang/tools/libclang/CXIndexDataConsumer.h +++ b/clang/tools/libclang/CXIndexDataConsumer.h @@ -460,8 +460,8 @@ private: const DeclContext *getEntityContainer(const Decl *D) const; - CXIdxClientFile getIndexFile(const FileEntry *File); - + CXIdxClientFile getIndexFile(OptionalFileEntryRef File); + CXIdxLoc getIndexLoc(SourceLocation Loc) const; void getEntityInfo(const NamedDecl *D, diff --git a/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp b/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp index f92afd8c3d84..4c3cb322eacf 100644 --- a/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp +++ b/clang/unittests/Analysis/FlowSensitive/TypeErasedDataflowAnalysisTest.cpp @@ -514,8 +514,17 @@ TEST_F(NoreturnDestructorTest, ConditionalOperatorNestedBranchReturns) { class SpecialBoolAnalysis final : public DataflowAnalysis { public: - explicit SpecialBoolAnalysis(ASTContext &Context) - : DataflowAnalysis(Context) {} + explicit SpecialBoolAnalysis(ASTContext &Context, Environment &Env) + : DataflowAnalysis(Context) { + Env.getDataflowAnalysisContext().setSyntheticFieldCallback( + [](QualType Ty) -> llvm::StringMap { + RecordDecl *RD = Ty->getAsRecordDecl(); + if (RD == nullptr || RD->getIdentifier() == nullptr || + RD->getName() != "SpecialBool") + return {}; + return {{"is_set", RD->getASTContext().BoolTy}}; + }); + } static NoopLattice initialElement() { return {}; } @@ -530,67 +539,18 @@ public: if (const auto *E = selectFirst( "call", match(cxxConstructExpr(HasSpecialBoolType).bind("call"), *S, getASTContext()))) { - cast(Env.getValue(*E)) - ->setProperty("is_set", Env.getBoolLiteralValue(false)); + Env.setValue(Env.getResultObjectLocation(*E).getSyntheticField("is_set"), + Env.getBoolLiteralValue(false)); } else if (const auto *E = selectFirst( "call", match(cxxMemberCallExpr(callee(cxxMethodDecl(ofClass( SpecialBoolRecordDecl)))) .bind("call"), *S, getASTContext()))) { - auto &ObjectLoc = - *cast(getImplicitObjectLocation(*E, Env)); - - refreshRecordValue(ObjectLoc, Env) - .setProperty("is_set", Env.getBoolLiteralValue(true)); + if (RecordStorageLocation *ObjectLoc = getImplicitObjectLocation(*E, Env)) + Env.setValue(ObjectLoc->getSyntheticField("is_set"), + Env.getBoolLiteralValue(true)); } } - - ComparisonResult compare(QualType Type, const Value &Val1, - const Environment &Env1, const Value &Val2, - const Environment &Env2) override { - const auto *Decl = Type->getAsCXXRecordDecl(); - if (Decl == nullptr || Decl->getIdentifier() == nullptr || - Decl->getName() != "SpecialBool") - return ComparisonResult::Unknown; - - auto *IsSet1 = cast_or_null(Val1.getProperty("is_set")); - auto *IsSet2 = cast_or_null(Val2.getProperty("is_set")); - if (IsSet1 == nullptr) - return IsSet2 == nullptr ? ComparisonResult::Same - : ComparisonResult::Different; - - if (IsSet2 == nullptr) - return ComparisonResult::Different; - - return Env1.proves(IsSet1->formula()) == Env2.proves(IsSet2->formula()) - ? ComparisonResult::Same - : ComparisonResult::Different; - } - - // Always returns `true` to accept the `MergedVal`. - bool merge(QualType Type, const Value &Val1, const Environment &Env1, - const Value &Val2, const Environment &Env2, Value &MergedVal, - Environment &MergedEnv) override { - const auto *Decl = Type->getAsCXXRecordDecl(); - if (Decl == nullptr || Decl->getIdentifier() == nullptr || - Decl->getName() != "SpecialBool") - return true; - - auto *IsSet1 = cast_or_null(Val1.getProperty("is_set")); - if (IsSet1 == nullptr) - return true; - - auto *IsSet2 = cast_or_null(Val2.getProperty("is_set")); - if (IsSet2 == nullptr) - return true; - - auto &IsSet = MergedEnv.makeAtomicBoolValue(); - MergedVal.setProperty("is_set", IsSet); - if (Env1.proves(IsSet1->formula()) && Env2.proves(IsSet2->formula())) - MergedEnv.assume(IsSet.formula()); - - return true; - } }; class JoinFlowConditionsTest : public Test { @@ -602,7 +562,7 @@ protected: AnalysisInputs( Code, ast_matchers::hasName("target"), [](ASTContext &Context, Environment &Env) { - return SpecialBoolAnalysis(Context); + return SpecialBoolAnalysis(Context, Env); }) .withASTBuildArgs({"-fsyntax-only", "-std=c++17"}), /*VerifyResults=*/[&Match](const llvm::StringMap< @@ -650,7 +610,9 @@ TEST_F(JoinFlowConditionsTest, JoinDistinctButProvablyEquivalentValues) { ASSERT_THAT(FooDecl, NotNull()); auto GetFoo = [FooDecl](const Environment &Env) -> const Formula & { - return cast(Env.getValue(*FooDecl)->getProperty("is_set")) + auto *Loc = + cast(Env.getStorageLocation(*FooDecl)); + return cast(Env.getValue(Loc->getSyntheticField("is_set"))) ->formula(); }; diff --git a/clang/unittests/Basic/FileEntryTest.cpp b/clang/unittests/Basic/FileEntryTest.cpp index dcd196417da7..f8a0b4a4edcd 100644 --- a/clang/unittests/Basic/FileEntryTest.cpp +++ b/clang/unittests/Basic/FileEntryTest.cpp @@ -92,24 +92,6 @@ TEST(FileEntryTest, FileEntryRef) { EXPECT_EQ(CE1, &R1.getFileEntry()); } -TEST(FileEntryTest, OptionalFileEntryRefDegradesToFileEntryPtr) { - FileEntryTestHelper Refs; - OptionalFileEntryRefDegradesToFileEntryPtr M0; - OptionalFileEntryRefDegradesToFileEntryPtr M1 = Refs.addFile("1"); - OptionalFileEntryRefDegradesToFileEntryPtr M2 = Refs.addFile("2"); - OptionalFileEntryRefDegradesToFileEntryPtr M0Also = std::nullopt; - OptionalFileEntryRefDegradesToFileEntryPtr M1Also = - Refs.addFileAlias("1-also", *M1); - - EXPECT_EQ(M0, M0Also); - EXPECT_EQ(StringRef("1"), M1->getName()); - EXPECT_EQ(StringRef("2"), M2->getName()); - EXPECT_EQ(StringRef("1-also"), M1Also->getName()); - - const FileEntry *CE1 = M1; - EXPECT_EQ(CE1, &M1->getFileEntry()); -} - TEST(FileEntryTest, equals) { FileEntryTestHelper Refs; FileEntryRef R1 = Refs.addFile("1"); @@ -126,13 +108,6 @@ TEST(FileEntryTest, equals) { EXPECT_NE(R1, R2); EXPECT_EQ(R1, R1Redirect); EXPECT_EQ(R1, R1Redirect2); - - OptionalFileEntryRefDegradesToFileEntryPtr M1 = R1; - - EXPECT_EQ(M1, &R1.getFileEntry()); - EXPECT_EQ(&R1.getFileEntry(), M1); - EXPECT_NE(M1, &R2.getFileEntry()); - EXPECT_NE(&R2.getFileEntry(), M1); } TEST(FileEntryTest, isSameRef) { diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp index 8782cb7c49ca..24b2fd599dc3 100644 --- a/clang/unittests/Format/FormatTest.cpp +++ b/clang/unittests/Format/FormatTest.cpp @@ -13950,6 +13950,19 @@ TEST_F(FormatTest, PullTrivialFunctionDefinitionsIntoSingleLine) { " void f() { int i; } \\\n" " int j;", getLLVMStyleWithColumns(23)); + + verifyFormat( + "void aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n" + " aaaaaaaaaaaaaaaaaa,\n" + " aaaaaaaaaaaaaaaaaabbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb) {}"); + + constexpr StringRef Code{"void foo() { /* Empty */ }"}; + verifyFormat(Code); + verifyFormat(Code, "void foo() { /* Empty */\n" + "}"); + verifyFormat(Code, "void foo() {\n" + "/* Empty */\n" + "}"); } TEST_F(FormatTest, PullEmptyFunctionDefinitionsIntoSingleLine) { diff --git a/clang/unittests/Format/FormatTestComments.cpp b/clang/unittests/Format/FormatTestComments.cpp index 9770d5090703..c249f4d9333f 100644 --- a/clang/unittests/Format/FormatTestComments.cpp +++ b/clang/unittests/Format/FormatTestComments.cpp @@ -386,15 +386,16 @@ TEST_F(FormatTestComments, UnderstandsBlockComments) { " /* Leading comment for bb... */ bbbbbbbbbbbbbbbbbbbbbbbbb);", format("f(aaaaaaaaaaaaaaaaaaaaaaaaa , \n" "/* Leading comment for bb... */ bbbbbbbbbbbbbbbbbbbbbbbbb);")); - EXPECT_EQ( + + verifyFormat( "void aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n" " aaaaaaaaaaaaaaaaaa,\n" - " aaaaaaaaaaaaaaaaaa) { /*aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa*/\n" - "}", - format("void aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n" - " aaaaaaaaaaaaaaaaaa ,\n" - " aaaaaaaaaaaaaaaaaa) { /*aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa*/\n" - "}")); + " aaaaaaaaaaaaaaaaaa) { /*aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa*/ }", + "void aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa(\n" + " aaaaaaaaaaaaaaaaaa ,\n" + " aaaaaaaaaaaaaaaaaa) { /*aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa*/\n" + "}"); + verifyFormat("f(/* aaaaaaaaaaaaaaaaaa = */\n" " aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa);"); diff --git a/clang/www/c_status.html b/clang/www/c_status.html index 91cae138074b..47acb1f87b8e 100644 --- a/clang/www/c_status.html +++ b/clang/www/c_status.html @@ -1151,7 +1151,7 @@ conformance.

Identifier syntax fixes N2939 - Unknown + Clang 15 Remove trigraphs??! diff --git a/clang/www/cxx_dr_status.html b/clang/www/cxx_dr_status.html index aafed2b6869e..d09cf616899a 100755 --- a/clang/www/cxx_dr_status.html +++ b/clang/www/cxx_dr_status.html @@ -2511,7 +2511,7 @@ of class templates 412 NAD Can a replacement allocation function be inline? - Yes + Clang 3.4 413 @@ -5117,7 +5117,7 @@ and POD class 873 C++11 Deducing rvalue references in declarative contexts - Yes + Clang 3.0 874 @@ -5519,7 +5519,7 @@ and POD class 952 CD6 Insufficient description of “naming class” - Yes + Clang 2.8 953 @@ -6473,7 +6473,7 @@ and POD class 1111 C++11 Remove dual-scope lookup of member template names - Yes + Clang 3.2 1112 @@ -7169,7 +7169,7 @@ and POD class 1227 CD3 Mixing immediate and non-immediate contexts in deduction failure - Yes + Clang 3.0 1228 @@ -7637,7 +7637,7 @@ and POD class 1305 CD3 alignof applied to array of unknown size - Yes + Clang 3.0 1306 @@ -7889,7 +7889,7 @@ and POD class 1347 CD3 Consistency of auto in multiple-declarator declarations - Yes + Clang 3.1 1348 @@ -7955,7 +7955,7 @@ and POD class 1358 CD3 Unintentionally ill-formed constexpr function template instances - Yes + Clang 3.1 1359 @@ -8681,7 +8681,7 @@ and POD class 1479 CD3 Literal operators and default arguments - Yes + Clang 3.1 1480 @@ -8699,7 +8699,7 @@ and POD class 1482 CD3 Point of declaration of enumeration - Yes + Clang 3.0 1483 @@ -9107,7 +9107,7 @@ and POD class 1550 CD3 Parenthesized throw-expression operand of conditional-expression - Yes + Clang 3.4 1551 @@ -9635,7 +9635,7 @@ and POD class 1638 CD4 Declaring an explicit specialization of a scoped enumeration - Yes + Clang 3.1 1639 @@ -10733,7 +10733,7 @@ and POD class 1821 CD6 Qualified redeclarations in a class member-specification - Yes + Clang 2.9 1822 @@ -11261,7 +11261,7 @@ and POD class 1909 CD4 Member class template with the same name as the class - Yes + Clang 3.7 1910 @@ -11447,7 +11447,7 @@ and POD class 1940 CD4 static_assert in anonymous unions - Yes + Clang 3.5 1941 @@ -11489,13 +11489,13 @@ and POD class 1947 NAD Digit separators following non-octal prefix - Yes + Clang 3.5 1948 NAD exception-specification of replacement global new - Yes + Clang 3.5 1949 @@ -14147,7 +14147,7 @@ and POD class 2390 CD5 Is the argument of __has_cpp_attribute macro-expanded? - Yes + Clang 14 2391 @@ -14243,7 +14243,7 @@ and POD class 2406 CD5 [[fallthrough]] attribute and iteration statements - Yes + Clang 5 2407 @@ -14903,7 +14903,7 @@ and POD class 2516 C++23 Locus of enum-specifier or opaque-enum-declaration - Yes + Clang 3.0 2517 @@ -15533,7 +15533,7 @@ and POD class 2621 C++23 Kind of lookup for using enum declarations - Yes + Clang 16 2622 @@ -15671,7 +15671,7 @@ and POD class 2644 C++23 Incorrect comment in example - Yes + Clang 8 2645 @@ -15707,7 +15707,7 @@ and POD class 2650 C++23 Incorrect example for ill-formed non-type template arguments - Yes + Clang 17 2651 diff --git a/compiler-rt/lib/asan/asan_mac.cpp b/compiler-rt/lib/asan/asan_mac.cpp index 5d5146e0cde0..1b0e9b3fe006 100644 --- a/compiler-rt/lib/asan/asan_mac.cpp +++ b/compiler-rt/lib/asan/asan_mac.cpp @@ -139,9 +139,11 @@ typedef void (*dispatch_mach_handler_function_t)(void *context, dispatch_mach_reason reason, dispatch_mach_msg_t message, mach_error_t error); +# if !defined(MISSING_BLOCKS_SUPPORT) typedef void (^dispatch_mach_handler_t)(dispatch_mach_reason reason, dispatch_mach_msg_t message, mach_error_t error); +# endif // A wrapper for the ObjC blocks used to support libdispatch. typedef struct { diff --git a/compiler-rt/lib/builtins/CMakeLists.txt b/compiler-rt/lib/builtins/CMakeLists.txt index 069d33bfd3d3..ea72c595a9b8 100644 --- a/compiler-rt/lib/builtins/CMakeLists.txt +++ b/compiler-rt/lib/builtins/CMakeLists.txt @@ -38,6 +38,13 @@ if (CMAKE_SOURCE_DIR STREQUAL CMAKE_CURRENT_SOURCE_DIR) include(UseLibtool) endif() include(AddCompilerRT) + + if(MINGW) + # Simplified version of what's set in cmake/config-ix.cmake; not including + # builtins, which are linked separately. + set(MINGW_LIBRARIES mingw32 moldname mingwex msvcrt advapi32 shell32 + user32 kernel32 mingw32 moldname mingwex msvcrt) + endif() endif() if (COMPILER_RT_STANDALONE_BUILD) @@ -881,12 +888,14 @@ if(COMPILER_RT_BUILD_STANDALONE_LIBATOMIC) endif() foreach (arch ${BUILTIN_SUPPORTED_ARCH}) if(CAN_TARGET_${arch}) + list(APPEND COMPILER_RT_LIBATOMIC_LINK_LIBS_${arch} clang_rt.builtins-${arch}) + append_list_if(MINGW "${MINGW_LIBRARIES}" COMPILER_RT_LIBATOMIC_LINK_LIBS_${arch}) add_compiler_rt_runtime(clang_rt.atomic ${BUILTIN_TYPE} ARCHS ${arch} SOURCES atomic.c LINK_FLAGS ${COMPILER_RT_LIBATOMIC_LINK_FLAGS} - LINK_LIBS clang_rt.builtins-${arch} + LINK_LIBS ${COMPILER_RT_LIBATOMIC_LINK_LIBS_${arch}} PARENT_TARGET builtins-standalone-atomic) endif() endforeach() diff --git a/compiler-rt/lib/hwasan/hwasan_flags.inc b/compiler-rt/lib/hwasan/hwasan_flags.inc index 978fa46b705c..058a0457b9e7 100644 --- a/compiler-rt/lib/hwasan/hwasan_flags.inc +++ b/compiler-rt/lib/hwasan/hwasan_flags.inc @@ -84,3 +84,10 @@ HWASAN_FLAG(bool, malloc_bisect_dump, false, // are untagged before the call. HWASAN_FLAG(bool, fail_without_syscall_abi, true, "Exit if fail to request relaxed syscall ABI.") + +HWASAN_FLAG( + uptr, fixed_shadow_base, -1, + "If not -1, HWASan will attempt to allocate the shadow at this address, " + "instead of choosing one dynamically." + "Tip: this can be combined with the compiler option, " + "-hwasan-mapping-offset, to optimize the instrumentation.") diff --git a/compiler-rt/lib/hwasan/hwasan_linux.cpp b/compiler-rt/lib/hwasan/hwasan_linux.cpp index 81226da976d1..f01fa4276413 100644 --- a/compiler-rt/lib/hwasan/hwasan_linux.cpp +++ b/compiler-rt/lib/hwasan/hwasan_linux.cpp @@ -106,8 +106,12 @@ static uptr GetHighMemEnd() { } static void InitializeShadowBaseAddress(uptr shadow_size_bytes) { - __hwasan_shadow_memory_dynamic_address = - FindDynamicShadowStart(shadow_size_bytes); + if (flags()->fixed_shadow_base != (uptr)-1) { + __hwasan_shadow_memory_dynamic_address = flags()->fixed_shadow_base; + } else { + __hwasan_shadow_memory_dynamic_address = + FindDynamicShadowStart(shadow_size_bytes); + } } static void MaybeDieIfNoTaggingAbi(const char *message) { diff --git a/compiler-rt/lib/interception/interception_win.cpp b/compiler-rt/lib/interception/interception_win.cpp index f3e62419beff..f1a549b938e2 100644 --- a/compiler-rt/lib/interception/interception_win.cpp +++ b/compiler-rt/lib/interception/interception_win.cpp @@ -578,6 +578,7 @@ static size_t GetInstructionSize(uptr address, size_t* rel_offset = nullptr) { case 0x018a: // mov al, byte ptr [rcx] return 2; + case 0x058A: // 8A 05 XX XX XX XX : mov al, byte ptr [XX XX XX XX] case 0x058B: // 8B 05 XX XX XX XX : mov eax, dword ptr [XX XX XX XX] if (rel_offset) *rel_offset = 2; diff --git a/compiler-rt/lib/orc/macho_platform.cpp b/compiler-rt/lib/orc/macho_platform.cpp index 73b17a0799c4..e3a1cdf3c4fc 100644 --- a/compiler-rt/lib/orc/macho_platform.cpp +++ b/compiler-rt/lib/orc/macho_platform.cpp @@ -352,36 +352,13 @@ private: Error requestPushSymbols(JITDylibState &JDS, span> Symbols); - /// Visits the symbol table for the JITDylib associated with DSOHandle. - /// Visitor should be callable as - /// - /// void (size_t, - /// std::optional>) - /// - /// The visitor function will be called for each element of the Symbols, but - /// in an arbitrary order. The first argument of the callback will indicate - /// the index of the result. The second argument will be std::nullopt (if the - /// symbol at the given index was not present in the symbol table), or a - /// pair containing the symbol's address and flags. - /// - /// This function will remove all elements of Symbols that are found, leaving - /// only the symbols that were not. This allows it to dovetail with - /// requestPushSymbols, enabling the following idiom: - /// - /// ... - /// visitSymbolAddrs(DSO, Symbols); - /// if (!Symbols.empty()) { - /// requestPushSymbols(DSO, Symbols); - /// visitSymbolAddrs(DSO, Symbols); - /// for (auto &Sym : Symbols) { - /// -- handle symbols that were not found -- - /// } - /// } - /// - template - void visitSymbolAddrs(JITDylibState &JDS, - std::vector> &Symbols, - VisitorFn &&Visit); + /// Attempts to look up the given symbols locally, requesting a push from the + /// remote if they're not found. Results are written to the Result span, which + /// must have the same size as the Symbols span. + Error + lookupSymbols(JITDylibState &JDS, std::unique_lock &JDStatesLock, + span> Result, + span> Symbols); bool lookupUnwindSections(void *Addr, unw_dynamic_unwind_sections &Info); @@ -830,7 +807,7 @@ int MachOPlatformRuntimeState::dlclose(void *DSOHandle) { } void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, const char *Symbol) { - std::lock_guard Lock(JDStatesMutex); + std::unique_lock Lock(JDStatesMutex); auto *JDS = getJITDylibStateByHeader(DSOHandle); if (!JDS) { std::ostringstream ErrStream; @@ -839,41 +816,16 @@ void *MachOPlatformRuntimeState::dlsym(void *DSOHandle, const char *Symbol) { return nullptr; } - std::string MangledName("_"); - MangledName += Symbol; - std::vector> Symbols; - Symbols.push_back({MangledName, false}); - - ExecutorAddr Result; - using ElemResult = - std::optional>; - - // Try to resolve the symbol in the local symbol tables. - visitSymbolAddrs(*JDS, Symbols, [&](size_t Idx, ElemResult E) { - if (E) - Result = E->first; - }); + std::string MangledName = std::string("_") + Symbol; + std::pair Lookup(MangledName, false); + std::pair Result; - // Return early if we found it. - if (Symbols.empty()) - return Result.toPtr(); - - // Otherwise call back to the controller to try to request that the symbol - // be materialized. - if (auto Err = requestPushSymbols(*JDS, {Symbols.data(), Symbols.size()})) { + if (auto Err = lookupSymbols(*JDS, Lock, {&Result, 1}, {&Lookup, 1})) { DLFcnError = toString(std::move(Err)); return nullptr; } - // Try another local resolution. - visitSymbolAddrs(*JDS, Symbols, [&](size_t Idx, ElemResult E) { - if (E) - Result = E->first; - }); - - // At this point Result has either been set (if we found the symbol) or is - // still null (if we didn't). Either way it's the right value. - return Result.toPtr(); + return Result.first.toPtr(); } int MachOPlatformRuntimeState::registerAtExit(void (*F)(void *), void *Arg, @@ -965,22 +917,71 @@ Error MachOPlatformRuntimeState::requestPushSymbols( return OpErr; } -template -void MachOPlatformRuntimeState::visitSymbolAddrs( - JITDylibState &JDS, std::vector> &Symbols, - VisitorFn &&Visit) { - - std::vector> RemainingSymbols; - +Error MachOPlatformRuntimeState::lookupSymbols( + JITDylibState &JDS, std::unique_lock &JDStatesLock, + span> Result, + span> Symbols) { + assert(JDStatesLock.owns_lock() && + "JDStatesLock should be locked at call-site"); + assert(Result.size() == Symbols.size() && + "Results and Symbols span sizes should match"); + + // Make an initial pass over the local symbol table. + std::vector MissingSymbolIndexes; for (size_t Idx = 0; Idx != Symbols.size(); ++Idx) { auto I = JDS.SymbolTable.find(Symbols[Idx].first); if (I != JDS.SymbolTable.end()) - Visit(Idx, I->second); + Result[Idx] = I->second; else - RemainingSymbols.push_back(Symbols[Idx]); + MissingSymbolIndexes.push_back(Idx); } - Symbols = std::move(RemainingSymbols); + // If everything has been resolved already then bail out early. + if (MissingSymbolIndexes.empty()) + return Error::success(); + + // Otherwise call back to the controller to try to request that the symbol + // be materialized. + std::vector> MissingSymbols; + MissingSymbols.reserve(MissingSymbolIndexes.size()); + ORC_RT_DEBUG({ + printdbg("requesting push of %i missing symbols...\n", + MissingSymbolIndexes.size()); + }); + for (auto MissingIdx : MissingSymbolIndexes) + MissingSymbols.push_back(Symbols[MissingIdx]); + + JDStatesLock.unlock(); + if (auto Err = requestPushSymbols( + JDS, {MissingSymbols.data(), MissingSymbols.size()})) + return Err; + JDStatesLock.lock(); + + // Try to resolve the previously missing symbols locally. + std::vector MissingRequiredSymbols; + for (auto MissingIdx : MissingSymbolIndexes) { + auto I = JDS.SymbolTable.find(Symbols[MissingIdx].first); + if (I != JDS.SymbolTable.end()) + Result[MissingIdx] = I->second; + else { + if (Symbols[MissingIdx].second) + MissingRequiredSymbols.push_back(MissingIdx); + else + Result[MissingIdx] = {ExecutorAddr(), {}}; + } + } + + // Error out if any missing symbols could not be resolved. + if (!MissingRequiredSymbols.empty()) { + std::ostringstream ErrStream; + ErrStream << "Lookup could not find required symbols: [ "; + for (auto MissingIdx : MissingRequiredSymbols) + ErrStream << "\"" << Symbols[MissingIdx].first << "\" "; + ErrStream << "]"; + return make_error(ErrStream.str()); + } + + return Error::success(); } // eh-frame registration functions. diff --git a/compiler-rt/lib/profile/InstrProfiling.h b/compiler-rt/lib/profile/InstrProfiling.h index c5b0b34f2d8a..137115996748 100644 --- a/compiler-rt/lib/profile/InstrProfiling.h +++ b/compiler-rt/lib/profile/InstrProfiling.h @@ -54,6 +54,12 @@ int __llvm_profile_is_continuous_mode_enabled(void); */ void __llvm_profile_enable_continuous_mode(void); +/*! + * \brief Disable continuous mode. + * + */ +void __llvm_profile_disable_continuous_mode(void); + /*! * \brief Set the page size. * diff --git a/compiler-rt/lib/profile/InstrProfilingBuffer.c b/compiler-rt/lib/profile/InstrProfilingBuffer.c index cd1f067bd188..af52804b2b53 100644 --- a/compiler-rt/lib/profile/InstrProfilingBuffer.c +++ b/compiler-rt/lib/profile/InstrProfilingBuffer.c @@ -33,6 +33,10 @@ COMPILER_RT_VISIBILITY void __llvm_profile_enable_continuous_mode(void) { ContinuouslySyncProfile = 1; } +COMPILER_RT_VISIBILITY void __llvm_profile_disable_continuous_mode(void) { + ContinuouslySyncProfile = 0; +} + COMPILER_RT_VISIBILITY void __llvm_profile_set_page_size(unsigned PS) { PageSize = PS; } diff --git a/compiler-rt/lib/profile/InstrProfilingFile.c b/compiler-rt/lib/profile/InstrProfilingFile.c index 1685b30b9492..745c567f2167 100644 --- a/compiler-rt/lib/profile/InstrProfilingFile.c +++ b/compiler-rt/lib/profile/InstrProfilingFile.c @@ -806,6 +806,7 @@ static int parseFilenamePattern(const char *FilenamePat, if (__llvm_profile_is_continuous_mode_enabled()) { PROF_WARN("%%c specifier can only be specified once in %s.\n", FilenamePat); + __llvm_profile_disable_continuous_mode(); return -1; } #if defined(__APPLE__) || defined(__ELF__) || defined(_WIN32) diff --git a/compiler-rt/lib/scudo/standalone/combined.h b/compiler-rt/lib/scudo/standalone/combined.h index 25ad11dbf7ee..65ddc488370a 100644 --- a/compiler-rt/lib/scudo/standalone/combined.h +++ b/compiler-rt/lib/scudo/standalone/combined.h @@ -14,6 +14,7 @@ #include "flags.h" #include "flags_parser.h" #include "local_cache.h" +#include "mem_map.h" #include "memtag.h" #include "options.h" #include "quarantine.h" @@ -935,8 +936,7 @@ public: uptr getRingBufferSize() { initThreadMaybe(); - auto *RingBuffer = getRingBuffer(); - return RingBuffer ? ringBufferSizeInBytes(RingBuffer->Size) : 0; + return RingBufferElements ? ringBufferSizeInBytes(RingBufferElements) : 0; } static bool setRingBufferSizeForBuffer(char *Buffer, size_t Size) { @@ -966,8 +966,9 @@ public: static void getErrorInfo(struct scudo_error_info *ErrorInfo, uintptr_t FaultAddr, const char *DepotPtr, const char *RegionInfoPtr, const char *RingBufferPtr, - const char *Memory, const char *MemoryTags, - uintptr_t MemoryAddr, size_t MemorySize) { + size_t RingBufferSize, const char *Memory, + const char *MemoryTags, uintptr_t MemoryAddr, + size_t MemorySize) { *ErrorInfo = {}; if (!allocatorSupportsMemoryTagging() || MemoryAddr + MemorySize < MemoryAddr) @@ -986,7 +987,7 @@ public: // Check the ring buffer. For primary allocations this will only find UAF; // for secondary allocations we can find either UAF or OOB. getRingBufferErrorInfo(ErrorInfo, NextErrorReport, FaultAddr, Depot, - RingBufferPtr); + RingBufferPtr, RingBufferSize); // Check for OOB in the 28 blocks surrounding the 3 we checked earlier. // Beyond that we are likely to hit false positives. @@ -1051,15 +1052,15 @@ private: atomic_u32 DeallocationTid; }; - MemMapT MemMap; atomic_uptr Pos; - u32 Size; // An array of Size (at least one) elements of type Entry is immediately // following to this struct. }; // Pointer to memory mapped area starting with AllocationRingBuffer struct, // and immediately followed by Size elements of type Entry. char *RawRingBuffer = {}; + u32 RingBufferElements = 0; + MemMapT RawRingBufferMap; // The following might get optimized out by the compiler. NOINLINE void performSanityChecks() { @@ -1267,7 +1268,7 @@ private: u32 DeallocationTid) { uptr Pos = atomic_fetch_add(&getRingBuffer()->Pos, 1, memory_order_relaxed); typename AllocationRingBuffer::Entry *Entry = - getRingBufferEntry(RawRingBuffer, Pos % getRingBuffer()->Size); + getRingBufferEntry(RawRingBuffer, Pos % RingBufferElements); // First invalidate our entry so that we don't attempt to interpret a // partially written state in getSecondaryErrorInfo(). The fences below @@ -1408,17 +1409,19 @@ private: size_t &NextErrorReport, uintptr_t FaultAddr, const StackDepot *Depot, - const char *RingBufferPtr) { + const char *RingBufferPtr, + size_t RingBufferSize) { auto *RingBuffer = reinterpret_cast(RingBufferPtr); - if (!RingBuffer || RingBuffer->Size == 0) + size_t RingBufferElements = ringBufferElementsFromBytes(RingBufferSize); + if (!RingBuffer || RingBufferElements == 0) return; uptr Pos = atomic_load_relaxed(&RingBuffer->Pos); - for (uptr I = Pos - 1; - I != Pos - 1 - RingBuffer->Size && NextErrorReport != NumErrorReports; + for (uptr I = Pos - 1; I != Pos - 1 - RingBufferElements && + NextErrorReport != NumErrorReports; --I) { - auto *Entry = getRingBufferEntry(RingBufferPtr, I % RingBuffer->Size); + auto *Entry = getRingBufferEntry(RingBufferPtr, I % RingBufferElements); uptr EntryPtr = atomic_load_relaxed(&Entry->Ptr); if (!EntryPtr) continue; @@ -1508,9 +1511,8 @@ private: getPageSizeCached()), "scudo:ring_buffer"); RawRingBuffer = reinterpret_cast(MemMap.getBase()); - auto *RingBuffer = reinterpret_cast(RawRingBuffer); - RingBuffer->MemMap = MemMap; - RingBuffer->Size = AllocationRingBufferSize; + RawRingBufferMap = MemMap; + RingBufferElements = AllocationRingBufferSize; static_assert(sizeof(AllocationRingBuffer) % alignof(typename AllocationRingBuffer::Entry) == 0, @@ -1520,16 +1522,23 @@ private: void unmapRingBuffer() { auto *RingBuffer = getRingBuffer(); if (RingBuffer != nullptr) { - MemMapT MemMap = RingBuffer->MemMap; - MemMap.unmap(MemMap.getBase(), MemMap.getCapacity()); + RawRingBufferMap.unmap(RawRingBufferMap.getBase(), + RawRingBufferMap.getCapacity()); } RawRingBuffer = nullptr; } - static constexpr size_t ringBufferSizeInBytes(u32 AllocationRingBufferSize) { + static constexpr size_t ringBufferSizeInBytes(u32 RingBufferElements) { return sizeof(AllocationRingBuffer) + - AllocationRingBufferSize * - sizeof(typename AllocationRingBuffer::Entry); + RingBufferElements * sizeof(typename AllocationRingBuffer::Entry); + } + + static constexpr size_t ringBufferElementsFromBytes(size_t Bytes) { + if (Bytes < sizeof(AllocationRingBuffer)) { + return 0; + } + return (Bytes - sizeof(AllocationRingBuffer)) / + sizeof(typename AllocationRingBuffer::Entry); } inline AllocationRingBuffer *getRingBuffer() { diff --git a/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp b/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp index 74456450a476..5b01ebe11c09 100644 --- a/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp +++ b/compiler-rt/lib/scudo/standalone/fuzz/get_error_info_fuzzer.cpp @@ -46,14 +46,11 @@ extern "C" int LLVMFuzzerTestOneInput(uint8_t *Data, size_t Size) { } std::string RingBufferBytes = FDP.ConsumeRemainingBytesAsString(); - // RingBuffer is too short. - if (!AllocatorT::setRingBufferSizeForBuffer(RingBufferBytes.data(), - RingBufferBytes.size())) - return 0; scudo_error_info ErrorInfo; AllocatorT::getErrorInfo(&ErrorInfo, FaultAddr, StackDepot.data(), - RegionInfo.data(), RingBufferBytes.data(), Memory, - MemoryTags, MemoryAddr, MemorySize); + RegionInfo.data(), RingBufferBytes.data(), + RingBufferBytes.size(), Memory, MemoryTags, + MemoryAddr, MemorySize); return 0; } diff --git a/compiler-rt/lib/scudo/standalone/include/scudo/interface.h b/compiler-rt/lib/scudo/standalone/include/scudo/interface.h index a664b9825f20..a2dedea910cc 100644 --- a/compiler-rt/lib/scudo/standalone/include/scudo/interface.h +++ b/compiler-rt/lib/scudo/standalone/include/scudo/interface.h @@ -20,6 +20,19 @@ __attribute__((weak)) const char *__scudo_default_options(void); __attribute__((weak)) void __scudo_allocate_hook(void *ptr, size_t size); __attribute__((weak)) void __scudo_deallocate_hook(void *ptr); +// `realloc` involves both deallocation and allocation but they are not reported +// atomically. In one specific case which may keep taking a snapshot right in +// the middle of `realloc` reporting the deallocation and allocation, it may +// confuse the user by missing memory from `realloc`. To alleviate that case, +// define the two `realloc` hooks to get the knowledge of the bundled hook +// calls. These hooks are optional and should only be used when a hooks user +// wants to track reallocs more closely. +// +// See more details in the comment of `realloc` in wrapper_c.inc. +__attribute__((weak)) void +__scudo_realloc_allocate_hook(void *old_ptr, void *new_ptr, size_t size); +__attribute__((weak)) void __scudo_realloc_deallocate_hook(void *old_ptr); + void __scudo_print_stats(void); typedef void (*iterate_callback)(uintptr_t base, size_t size, void *arg); diff --git a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp index 150688b5b70a..f5e17d721486 100644 --- a/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp +++ b/compiler-rt/lib/scudo/standalone/tests/wrappers_c_test.cpp @@ -61,8 +61,14 @@ struct AllocContext { struct DeallocContext { void *Ptr; }; +struct ReallocContext { + void *AllocPtr; + void *DeallocPtr; + size_t Size; +}; static AllocContext AC; static DeallocContext DC; +static ReallocContext RC; #if (SCUDO_ENABLE_HOOKS_TESTS == 1) __attribute__((visibility("default"))) void __scudo_allocate_hook(void *Ptr, @@ -73,6 +79,28 @@ __attribute__((visibility("default"))) void __scudo_allocate_hook(void *Ptr, __attribute__((visibility("default"))) void __scudo_deallocate_hook(void *Ptr) { DC.Ptr = Ptr; } +__attribute__((visibility("default"))) void +__scudo_realloc_allocate_hook(void *OldPtr, void *NewPtr, size_t Size) { + // Verify that __scudo_realloc_deallocate_hook is called first and set the + // right pointer. + EXPECT_EQ(OldPtr, RC.DeallocPtr); + RC.AllocPtr = NewPtr; + RC.Size = Size; + + // Note that this is only used for testing. In general, only one pair of hooks + // will be invoked in `realloc`. if __scudo_realloc_*_hook are not defined, + // it'll call the general hooks only. To make the test easier, we call the + // general one here so that either case (whether __scudo_realloc_*_hook are + // defined) will be verified without separating them into different tests. + __scudo_allocate_hook(NewPtr, Size); +} +__attribute__((visibility("default"))) void +__scudo_realloc_deallocate_hook(void *Ptr) { + RC.DeallocPtr = Ptr; + + // See the comment in the __scudo_realloc_allocate_hook above. + __scudo_deallocate_hook(Ptr); +} #endif // (SCUDO_ENABLE_HOOKS_TESTS == 1) } @@ -88,6 +116,7 @@ protected: void *InvalidPtr = reinterpret_cast(0xdeadbeef); AC.Ptr = InvalidPtr; DC.Ptr = InvalidPtr; + RC.AllocPtr = RC.DeallocPtr = InvalidPtr; } } void verifyAllocHookPtr(UNUSED void *Ptr) { @@ -102,6 +131,13 @@ protected: if (SCUDO_ENABLE_HOOKS_TESTS) EXPECT_EQ(Ptr, DC.Ptr); } + void verifyReallocHookPtrs(UNUSED void *OldPtr, void *NewPtr, size_t Size) { + if (SCUDO_ENABLE_HOOKS_TESTS) { + EXPECT_EQ(OldPtr, RC.DeallocPtr); + EXPECT_EQ(NewPtr, RC.AllocPtr); + EXPECT_EQ(Size, RC.Size); + } + } }; using ScudoWrappersCDeathTest = ScudoWrappersCTest; @@ -297,6 +333,7 @@ TEST_F(ScudoWrappersCDeathTest, Realloc) { verifyAllocHookSize(Size * 2U); verifyDeallocHookPtr(OldP); } + verifyReallocHookPtrs(OldP, P, Size * 2U); invalidateHookPtrs(); OldP = P; @@ -312,6 +349,7 @@ TEST_F(ScudoWrappersCDeathTest, Realloc) { verifyAllocHookPtr(P); verifyAllocHookSize(Size / 2U); } + verifyReallocHookPtrs(OldP, P, Size / 2U); free(P); EXPECT_DEATH(P = realloc(P, Size), ""); diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c.inc b/compiler-rt/lib/scudo/standalone/wrappers_c.inc index 0413ea49eac0..56d8ef20156e 100644 --- a/compiler-rt/lib/scudo/standalone/wrappers_c.inc +++ b/compiler-rt/lib/scudo/standalone/wrappers_c.inc @@ -27,6 +27,24 @@ static void reportDeallocation(void *ptr) { if (__scudo_deallocate_hook) __scudo_deallocate_hook(ptr); } +static void reportReallocAllocation(void *old_ptr, void *new_ptr, size_t size) { + DCHECK_NE(new_ptr, nullptr); + + if (SCUDO_ENABLE_HOOKS) { + if (__scudo_realloc_allocate_hook) + __scudo_realloc_allocate_hook(old_ptr, new_ptr, size); + else if (__scudo_allocate_hook) + __scudo_allocate_hook(new_ptr, size); + } +} +static void reportReallocDeallocation(void *old_ptr) { + if (SCUDO_ENABLE_HOOKS) { + if (__scudo_realloc_deallocate_hook) + __scudo_realloc_deallocate_hook(old_ptr); + else if (__scudo_deallocate_hook) + __scudo_deallocate_hook(old_ptr); + } +} extern "C" { @@ -183,16 +201,17 @@ INTERFACE WEAK void *SCUDO_PREFIX(realloc)(void *ptr, size_t size) { // new pointer. Before the reporting of both operations has been done, another // thread may get the old pointer from `malloc`. It may be misinterpreted as // double-use if it's not handled properly on the hook side. - reportDeallocation(ptr); + reportReallocDeallocation(ptr); void *NewPtr = SCUDO_ALLOCATOR.reallocate(ptr, size, SCUDO_MALLOC_ALIGNMENT); if (NewPtr != nullptr) { // Note that even if NewPtr == ptr, the size has changed. We still need to // report the new size. - reportAllocation(NewPtr, size); + reportReallocAllocation(/*OldPtr=*/ptr, NewPtr, size); } else { // If `realloc` fails, the old pointer is not released. Report the old - // pointer as allocated back. - reportAllocation(ptr, SCUDO_ALLOCATOR.getAllocSize(ptr)); + // pointer as allocated again. + reportReallocAllocation(/*OldPtr=*/ptr, /*NewPtr=*/ptr, + SCUDO_ALLOCATOR.getAllocSize(ptr)); } return scudo::setErrnoOnNull(NewPtr); diff --git a/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp b/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp index f203615ab360..21694c3f17fe 100644 --- a/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp +++ b/compiler-rt/lib/scudo/standalone/wrappers_c_bionic.cpp @@ -44,10 +44,9 @@ INTERFACE void __scudo_get_error_info( const char *ring_buffer, size_t ring_buffer_size, const char *memory, const char *memory_tags, uintptr_t memory_addr, size_t memory_size) { (void)(stack_depot_size); - (void)(ring_buffer_size); Allocator.getErrorInfo(error_info, fault_addr, stack_depot, region_info, - ring_buffer, memory, memory_tags, memory_addr, - memory_size); + ring_buffer, ring_buffer_size, memory, memory_tags, + memory_addr, memory_size); } INTERFACE const char *__scudo_get_stack_depot_addr() { diff --git a/compiler-rt/test/hwasan/TestCases/Linux/fixed-shadow.c b/compiler-rt/test/hwasan/TestCases/Linux/fixed-shadow.c new file mode 100644 index 000000000000..4ff1d3e64c1d --- /dev/null +++ b/compiler-rt/test/hwasan/TestCases/Linux/fixed-shadow.c @@ -0,0 +1,76 @@ +// Test fixed shadow base functionality. +// +// Default compiler instrumentation works with any shadow base (dynamic or fixed). +// RUN: %clang_hwasan %s -o %t && %run %t +// RUN: %clang_hwasan %s -o %t && HWASAN_OPTIONS=fixed_shadow_base=263878495698944 %run %t +// RUN: %clang_hwasan %s -o %t && HWASAN_OPTIONS=fixed_shadow_base=4398046511104 %run %t +// +// If -hwasan-mapping-offset is set, then the fixed_shadow_base needs to match. +// RUN: %clang_hwasan %s -mllvm -hwasan-mapping-offset=263878495698944 -o %t && HWASAN_OPTIONS=fixed_shadow_base=263878495698944 %run %t +// RUN: %clang_hwasan %s -mllvm -hwasan-mapping-offset=4398046511104 -o %t && HWASAN_OPTIONS=fixed_shadow_base=4398046511104 %run %t +// RUN: %clang_hwasan %s -mllvm -hwasan-mapping-offset=263878495698944 -o %t && HWASAN_OPTIONS=fixed_shadow_base=4398046511104 not %run %t +// RUN: %clang_hwasan %s -mllvm -hwasan-mapping-offset=4398046511104 -o %t && HWASAN_OPTIONS=fixed_shadow_base=263878495698944 not %run %t +// +// Note: if fixed_shadow_base is not set, compiler-rt will dynamically choose a +// shadow base, which has a tiny but non-zero probability of matching the +// compiler instrumentation. To avoid test flake, we do not test this case. +// +// Assume 48-bit VMA +// REQUIRES: aarch64-target-arch +// +// REQUIRES: Clang +// +// UNSUPPORTED: android + +#include +#include +#include +#include +#include +#include + +int main() { + __hwasan_enable_allocator_tagging(); + + // We test that the compiler instrumentation is able to access shadow memory + // for many different addresses. If we only test a small number of addresses, + // it might work by chance even if the shadow base does not match between the + // compiler instrumentation and compiler-rt. + void **mmaps[256]; + // 48-bit VMA + for (int i = 0; i < 256; i++) { + unsigned long long addr = (i * (1ULL << 40)); + + void *p = mmap((void *)addr, 4096, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + // We don't use MAP_FIXED, to avoid overwriting critical memory. + // However, if we don't get allocated the requested address, it + // isn't a useful test. + if ((unsigned long long)p != addr) { + munmap(p, 4096); + mmaps[i] = MAP_FAILED; + } else { + mmaps[i] = p; + } + } + + int failures = 0; + for (int i = 0; i < 256; i++) { + if (mmaps[i] == MAP_FAILED) { + failures++; + } else { + printf("%d %p\n", i, mmaps[i]); + munmap(mmaps[i], 4096); + } + } + + // We expect roughly 17 failures: + // - the page at address zero + // - 16 failures because the shadow memory takes up 1/16th of the address space + // We could also get unlucky e.g., if libraries or binaries are loaded into the + // exact addresses where we tried to map. + // To avoid test flake, we allow some margin of error. + printf("Failed: %d\n", failures); + assert(failures < 48); + return 0; +} diff --git a/compiler-rt/test/orc/TestCases/Darwin/arm64/Inputs/ret_self.S b/compiler-rt/test/orc/TestCases/Darwin/arm64/Inputs/ret_self.S new file mode 100644 index 000000000000..fce02b2182ab --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Darwin/arm64/Inputs/ret_self.S @@ -0,0 +1,11 @@ + .section __TEXT,__text,regular,pure_instructions + .build_version macos, 14, 0 sdk_version 14, 4 + .globl _ret_self + .p2align 2 +_ret_self: + adrp x0, _ret_self@PAGE + add x0, x0, _ret_self@PAGEOFF + ret + +.subsections_via_symbols + diff --git a/compiler-rt/test/orc/TestCases/Darwin/arm64/trivial-dlsym.c b/compiler-rt/test/orc/TestCases/Darwin/arm64/trivial-dlsym.c new file mode 100644 index 000000000000..b597f5dc2fe6 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Darwin/arm64/trivial-dlsym.c @@ -0,0 +1,44 @@ +// Test that __orc_rt_macho_jit_dlsym works as expected. +// +// RUN: %clang -c -o %t.sym.o %p/Inputs/ret_self.S +// RUN: %clang -c -o %t.test.o %s +// RUN: %llvm_jitlink \ +// RUN: -alias Platform:_dlopen=___orc_rt_macho_jit_dlopen \ +// RUN: -alias Platform:_dlsym=___orc_rt_macho_jit_dlsym \ +// RUN: -alias Platform:_dlclose=___orc_rt_macho_jit_dlclose \ +// RUN: %t.test.o -lextra_sym -jd extra_sym %t.sym.o | FileCheck %s + +// CHECK: entering main +// CHECK-NEXT: found "ret_self" at +// CHECK-NEXT: address of "ret_self" is consistent +// CHECK-NEXT: leaving main + +int printf(const char *restrict format, ...); +void *dlopen(const char *path, int mode); +void *dlsym(void *handle, const char *symbol); +int dlclose(void *handle); + +int main(int argc, char *argv[]) { + printf("entering main\n"); + void *H = dlopen("extra_sym", 0); + if (!H) { + printf("failed\n"); + return -1; + } + + void *(*ret_self)(void) = (void *(*)(void))dlsym(H, "ret_self"); + if (ret_self) + printf("found \"ret_self\" at %p\n", ret_self); + else + printf("failed to find \"ret_self\" via dlsym\n"); + + printf("address of \"ret_self\" is %s\n", + ret_self() == ret_self ? "consistent" : "inconsistent"); + + if (dlclose(H) == -1) { + printf("failed\n"); + return -1; + } + printf("leaving main\n"); + return 0; +} diff --git a/compiler-rt/test/orc/TestCases/Darwin/x86-64/Inputs/ret_self.S b/compiler-rt/test/orc/TestCases/Darwin/x86-64/Inputs/ret_self.S new file mode 100644 index 000000000000..bcea9868f18d --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Darwin/x86-64/Inputs/ret_self.S @@ -0,0 +1,12 @@ +// A function that returns its own address. Handy for testing whether JIT'd code +// and JIT symbol tables agree on addresses. + + .section __TEXT,__text,regular,pure_instructions + .build_version macos, 14, 0 + .globl _ret_self + .p2align 4, 0x90 +_ret_self: + leaq _ret_self(%rip), %rax + retq + +.subsections_via_symbols diff --git a/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-dlsym.c b/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-dlsym.c new file mode 100644 index 000000000000..b597f5dc2fe6 --- /dev/null +++ b/compiler-rt/test/orc/TestCases/Darwin/x86-64/trivial-dlsym.c @@ -0,0 +1,44 @@ +// Test that __orc_rt_macho_jit_dlsym works as expected. +// +// RUN: %clang -c -o %t.sym.o %p/Inputs/ret_self.S +// RUN: %clang -c -o %t.test.o %s +// RUN: %llvm_jitlink \ +// RUN: -alias Platform:_dlopen=___orc_rt_macho_jit_dlopen \ +// RUN: -alias Platform:_dlsym=___orc_rt_macho_jit_dlsym \ +// RUN: -alias Platform:_dlclose=___orc_rt_macho_jit_dlclose \ +// RUN: %t.test.o -lextra_sym -jd extra_sym %t.sym.o | FileCheck %s + +// CHECK: entering main +// CHECK-NEXT: found "ret_self" at +// CHECK-NEXT: address of "ret_self" is consistent +// CHECK-NEXT: leaving main + +int printf(const char *restrict format, ...); +void *dlopen(const char *path, int mode); +void *dlsym(void *handle, const char *symbol); +int dlclose(void *handle); + +int main(int argc, char *argv[]) { + printf("entering main\n"); + void *H = dlopen("extra_sym", 0); + if (!H) { + printf("failed\n"); + return -1; + } + + void *(*ret_self)(void) = (void *(*)(void))dlsym(H, "ret_self"); + if (ret_self) + printf("found \"ret_self\" at %p\n", ret_self); + else + printf("failed to find \"ret_self\" via dlsym\n"); + + printf("address of \"ret_self\" is %s\n", + ret_self() == ret_self ? "consistent" : "inconsistent"); + + if (dlclose(H) == -1) { + printf("failed\n"); + return -1; + } + printf("leaving main\n"); + return 0; +} diff --git a/compiler-rt/test/profile/ContinuousSyncMode/reset-default-profile.c b/compiler-rt/test/profile/ContinuousSyncMode/reset-default-profile.c new file mode 100644 index 000000000000..75af7684161c --- /dev/null +++ b/compiler-rt/test/profile/ContinuousSyncMode/reset-default-profile.c @@ -0,0 +1,21 @@ +// REQUIRES: darwin || linux + +// Test when LLVM_PROFILE_FILE is set incorrectly, it should fall backs to use default.profraw without runtime error. + +// Create & cd into a temporary directory. +// RUN: rm -rf %t.dir && mkdir -p %t.dir && cd %t.dir +// RUN: %clang -fprofile-instr-generate -fcoverage-mapping -mllvm -runtime-counter-relocation=true -o %t.exe %s +// RUN: env LLVM_PROFILE_FILE="incorrect-profile-name%m%c%c.profraw" %run %t.exe +// RUN: ls -l | FileCheck %s + +// CHECK: default.profraw +// CHECK-NOT: incorrect-profile-name.profraw + +#include +int f() { return 0; } + +int main(int argc, char **argv) { + FILE *File = fopen("default.profraw", "w"); + f(); + return 0; +} diff --git a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py index 50a97e2b77d1..0e20cfbbd264 100644 --- a/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py +++ b/cross-project-tests/debuginfo-tests/dexter/dex/debugger/visualstudio/VisualStudio.py @@ -276,6 +276,13 @@ class VisualStudio( project.Properties, "ActiveConfiguration" ).Object ActiveConfiguration.DebugSettings.CommandArguments = cmdline_str + ConfigurationName = ActiveConfiguration.ConfigurationName + SolConfig = self._fetch_property( + self._interface.Solution.SolutionBuild.SolutionConfigurations, + ConfigurationName, + ) + for Context in SolConfig.SolutionContexts: + Context.ShouldBuild = False self.context.logger.note("Launching VS debugger...") self._fn_go(False) diff --git a/flang/include/flang/Runtime/character.h b/flang/include/flang/Runtime/character.h index 24f26920bdd2..768de75b639c 100644 --- a/flang/include/flang/Runtime/character.h +++ b/flang/include/flang/Runtime/character.h @@ -44,17 +44,6 @@ void RTNAME(CharacterConcatenate)(Descriptor &accumulator, void RTNAME(CharacterConcatenateScalar1)( Descriptor &accumulator, const char *from, std::size_t chars); -// Copies the value(s) of 'rhs' to 'lhs'. Handles reallocation, -// truncation, or padding ss necessary. Crashes when not conforming and -// the LHS is not allocatable. Assumes independence of data. -// The LHS and RHS need not have the same kind of character; -// so when the LHS is a deallocated allocatable temporary result, this -// function can be used as a simple conversion routine. -// Call MoveAlloc() instead as an optimization when a temporary value is -// being assigned to a deferred-length allocatable. -void RTNAME(CharacterAssign)(Descriptor &lhs, const Descriptor &rhs, - const char *sourceFile = nullptr, int sourceLine = 0); - // CHARACTER comparisons. The kinds must match. Like std::memcmp(), // the result is less than zero, zero, or greater than zero if the first // argument is less than the second, equal to the second, or greater than diff --git a/flang/include/flang/Runtime/iostat.h b/flang/include/flang/Runtime/iostat.h index 0456e24f4e38..afce509cf1f5 100644 --- a/flang/include/flang/Runtime/iostat.h +++ b/flang/include/flang/Runtime/iostat.h @@ -85,6 +85,7 @@ enum Iostat { IostatBadOpOnChildUnit, IostatBadNewUnit, IostatBadListDirectedInputSeparator, + IostatNonExternalDefinedUnformattedIo, }; const char *IostatErrorString(int); diff --git a/flang/lib/Frontend/FrontendActions.cpp b/flang/lib/Frontend/FrontendActions.cpp index b114c552b552..d4a3e164d207 100644 --- a/flang/lib/Frontend/FrontendActions.cpp +++ b/flang/lib/Frontend/FrontendActions.cpp @@ -58,6 +58,7 @@ #include "llvm/Passes/PassBuilder.h" #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/StandardInstrumentations.h" +#include "llvm/Support/AMDGPUAddrSpace.h" #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/FileSystem.h" @@ -166,12 +167,6 @@ static void addAMDGPUSpecificMLIRItems(mlir::ModuleOp &mlirModule, const llvm::Triple triple(targetOpts.triple); const llvm::StringRef codeObjectVersionGlobalOpName = "__oclc_ABI_version"; - // TODO: Share address spaces enumeration between Clang and Flang. - // Currently this enumeration is defined in Clang specific class - // defined in file: clang/lib/Basic/Targets/AMDGPU.h . - // and we need to move it to LLVM directory. - const int constantAddressSpace = 4; - if (!triple.isAMDGPU()) { return; } @@ -202,7 +197,7 @@ static void addAMDGPUSpecificMLIRItems(mlir::ModuleOp &mlirModule, originalGVOp.setValueAttr( builder.getIntegerAttr(int32Type, oclcABIVERsion)); originalGVOp.setUnnamedAddr(mlir::LLVM::UnnamedAddr::Local); - originalGVOp.setAddrSpace(constantAddressSpace); + originalGVOp.setAddrSpace(llvm::AMDGPUAS::CONSTANT_ADDRESS); originalGVOp.setVisibility_(mlir::LLVM::Visibility::Hidden); return; } @@ -213,7 +208,7 @@ static void addAMDGPUSpecificMLIRItems(mlir::ModuleOp &mlirModule, /* Name */ codeObjectVersionGlobalOpName, /* Value */ builder.getIntegerAttr(int32Type, oclcABIVERsion)); covInfo.setUnnamedAddr(mlir::LLVM::UnnamedAddr::Local); - covInfo.setAddrSpace(constantAddressSpace); + covInfo.setAddrSpace(llvm::AMDGPUAS::CONSTANT_ADDRESS); covInfo.setVisibility_(mlir::LLVM::Visibility::Hidden); builder.setInsertionPointToStart(mlirModule.getBody()); builder.insert(covInfo); @@ -715,8 +710,8 @@ void CodeGenAction::lowerHLFIRToFIR() { // TODO: We should get this from TargetInfo. However, that depends on // too much of clang, so for now, replicate the functionality. static std::optional> -getVScaleRange(CompilerInstance &ci, - const Fortran::frontend::LangOptions &langOpts) { +getVScaleRange(CompilerInstance &ci) { + const auto &langOpts = ci.getInvocation().getLangOpts(); if (langOpts.VScaleMin || langOpts.VScaleMax) return std::pair( langOpts.VScaleMin ? langOpts.VScaleMin : 1, langOpts.VScaleMax); @@ -751,13 +746,9 @@ void CodeGenAction::generateLLVMIR() { const auto targetOpts = ci.getInvocation().getTargetOpts(); const llvm::Triple triple(targetOpts.triple); - // Only get the vscale range if AArch64. - if (triple.isAArch64()) { - auto langOpts = ci.getInvocation().getLangOpts(); - if (auto vsr = getVScaleRange(ci, langOpts)) { - config.VScaleMin = vsr->first; - config.VScaleMax = vsr->second; - } + if (auto vsr = getVScaleRange(ci)) { + config.VScaleMin = vsr->first; + config.VScaleMax = vsr->second; } // Create the pass pipeline diff --git a/flang/lib/Lower/OpenMP.cpp b/flang/lib/Lower/OpenMP.cpp index 0fa1ac76d57e..eeba87fcd151 100644 --- a/flang/lib/Lower/OpenMP.cpp +++ b/flang/lib/Lower/OpenMP.cpp @@ -2150,13 +2150,7 @@ static void createBodyOfOp( firOpBuilder, eval.getNestedEvaluations()); // Insert the terminator. - if constexpr (std::is_same_v || - std::is_same_v) { - mlir::ValueRange results; - firOpBuilder.create(loc, results); - } else { - firOpBuilder.create(loc); - } + Fortran::lower::genOpenMPTerminator(firOpBuilder, op.getOperation(), loc); // Reset the insert point to before the terminator. resetBeforeTerminator(firOpBuilder, storeOp, block); @@ -3233,7 +3227,6 @@ static void genOMP(Fortran::lower::AbstractConverter &converter, Fortran::lower::pft::Evaluation &eval, const Fortran::parser::OpenMPSectionConstruct §ionConstruct) { - fir::FirOpBuilder &firOpBuilder = converter.getFirOpBuilder(); mlir::Location currentLocation = converter.getCurrentLocation(); const Fortran::parser::OpenMPConstruct *parentOmpConstruct = eval.parentConstruct->getIf(); @@ -3251,10 +3244,9 @@ genOMP(Fortran::lower::AbstractConverter &converter, .t); // Currently only private/firstprivate clause is handled, and // all privatization is done within `omp.section` operations. - mlir::omp::SectionOp sectionOp = - firOpBuilder.create(currentLocation); - createBodyOfOp(sectionOp, converter, currentLocation, - eval, §ionsClauseList); + genOpWithBody(converter, eval, currentLocation, + /*outerCombined=*/false, + §ionsClauseList); } static void diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp index 0b1e57e8f6c3..17efa45b8667 100644 --- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp +++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp @@ -1132,8 +1132,13 @@ hlfir::genTypeAndKindConvert(mlir::Location loc, fir::FirOpBuilder &builder, std::optional toKindCharConvert; if (auto toCharTy = mlir::dyn_cast(toType)) { if (auto fromCharTy = mlir::dyn_cast(fromType)) - if (toCharTy.getFKind() != fromCharTy.getFKind()) + if (toCharTy.getFKind() != fromCharTy.getFKind()) { toKindCharConvert = toCharTy.getFKind(); + // Preserve source length (padding/truncation will occur in assignment + // if needed). + toType = fir::CharacterType::get( + fromType.getContext(), toCharTy.getFKind(), fromCharTy.getLen()); + } // Do not convert in case of character length mismatch only, hlfir.assign // deals with it. if (!toKindCharConvert) diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp index bf175c8ebade..293208ce3b60 100644 --- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp +++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp @@ -2387,6 +2387,9 @@ struct XArrayCoorOpConversion const bool baseIsBoxed = coor.getMemref().getType().isa(); TypePair baseBoxTyPair = baseIsBoxed ? getBoxTypePair(coor.getMemref().getType()) : TypePair{}; + mlir::LLVM::IntegerOverflowFlagsAttr nsw = + mlir::LLVM::IntegerOverflowFlagsAttr::get( + rewriter.getContext(), mlir::LLVM::IntegerOverflowFlags::nsw); // For each dimension of the array, generate the offset calculation. for (unsigned i = 0; i < rank; ++i, ++indexOffset, ++shapeOffset, @@ -2407,14 +2410,15 @@ struct XArrayCoorOpConversion if (normalSlice) step = integerCast(loc, rewriter, idxTy, operands[sliceOffset + 2]); } - auto idx = rewriter.create(loc, idxTy, index, lb); + auto idx = rewriter.create(loc, idxTy, index, lb, nsw); mlir::Value diff = - rewriter.create(loc, idxTy, idx, step); + rewriter.create(loc, idxTy, idx, step, nsw); if (normalSlice) { mlir::Value sliceLb = integerCast(loc, rewriter, idxTy, operands[sliceOffset]); - auto adj = rewriter.create(loc, idxTy, sliceLb, lb); - diff = rewriter.create(loc, idxTy, diff, adj); + auto adj = + rewriter.create(loc, idxTy, sliceLb, lb, nsw); + diff = rewriter.create(loc, idxTy, diff, adj, nsw); } // Update the offset given the stride and the zero based index `diff` // that was just computed. @@ -2422,17 +2426,21 @@ struct XArrayCoorOpConversion // Use stride in bytes from the descriptor. mlir::Value stride = getStrideFromBox(loc, baseBoxTyPair, operands[0], i, rewriter); - auto sc = rewriter.create(loc, idxTy, diff, stride); - offset = rewriter.create(loc, idxTy, sc, offset); + auto sc = + rewriter.create(loc, idxTy, diff, stride, nsw); + offset = + rewriter.create(loc, idxTy, sc, offset, nsw); } else { // Use stride computed at last iteration. - auto sc = rewriter.create(loc, idxTy, diff, prevExt); - offset = rewriter.create(loc, idxTy, sc, offset); + auto sc = + rewriter.create(loc, idxTy, diff, prevExt, nsw); + offset = + rewriter.create(loc, idxTy, sc, offset, nsw); // Compute next stride assuming contiguity of the base array // (in element number). auto nextExt = integerCast(loc, rewriter, idxTy, operands[shapeOffset]); - prevExt = - rewriter.create(loc, idxTy, prevExt, nextExt); + prevExt = rewriter.create(loc, idxTy, prevExt, + nextExt, nsw); } } @@ -2491,8 +2499,8 @@ struct XArrayCoorOpConversion assert(coor.getLenParams().size() == 1); auto length = integerCast(loc, rewriter, idxTy, operands[coor.lenParamsOffset()]); - offset = - rewriter.create(loc, idxTy, offset, length); + offset = rewriter.create(loc, idxTy, offset, + length, nsw); } else { TODO(loc, "compute size of derived type with type parameters"); } @@ -2665,6 +2673,9 @@ private: auto cpnTy = fir::dyn_cast_ptrOrBoxEleTy(boxObjTy); mlir::Type llvmPtrTy = ::getLlvmPtrType(coor.getContext()); mlir::Type byteTy = ::getI8Type(coor.getContext()); + mlir::LLVM::IntegerOverflowFlagsAttr nsw = + mlir::LLVM::IntegerOverflowFlagsAttr::get( + rewriter.getContext(), mlir::LLVM::IntegerOverflowFlags::nsw); for (unsigned i = 1, last = operands.size(); i < last; ++i) { if (auto arrTy = cpnTy.dyn_cast()) { @@ -2680,9 +2691,9 @@ private: index < lastIndex; ++index) { mlir::Value stride = getStrideFromBox(loc, boxTyPair, operands[0], index - i, rewriter); - auto sc = rewriter.create(loc, idxTy, - operands[index], stride); - off = rewriter.create(loc, idxTy, sc, off); + auto sc = rewriter.create( + loc, idxTy, operands[index], stride, nsw); + off = rewriter.create(loc, idxTy, sc, off, nsw); } resultAddr = rewriter.create( loc, llvmPtrTy, byteTy, resultAddr, diff --git a/flang/lib/Parser/prescan.cpp b/flang/lib/Parser/prescan.cpp index 449ea6014442..79cdaccf1fbf 100644 --- a/flang/lib/Parser/prescan.cpp +++ b/flang/lib/Parser/prescan.cpp @@ -706,6 +706,7 @@ void Prescanner::QuotedCharacterLiteral( char quote{*at_}; const char *end{at_ + 1}; inCharLiteral_ = true; + continuationInCharLiteral_ = true; const auto emit{[&](char ch) { EmitChar(tokens, ch); }}; const auto insert{[&](char ch) { EmitInsertedChar(tokens, ch); }}; bool isEscaped{false}; @@ -749,16 +750,9 @@ void Prescanner::QuotedCharacterLiteral( break; } inCharLiteral_ = true; - if (insertASpace_) { - if (features_.ShouldWarn( - common::LanguageFeature::MiscSourceExtensions)) { - Say(GetProvenanceRange(at_, end), - "Repeated quote mark in character literal continuation line should have been preceded by '&'"_port_en_US); - } - insertASpace_ = false; - } } } + continuationInCharLiteral_ = false; inCharLiteral_ = false; } @@ -1122,7 +1116,15 @@ const char *Prescanner::FreeFormContinuationLine(bool ampersand) { } else if (*p == '!' || *p == '\n' || *p == '#') { return nullptr; } else if (ampersand || IsImplicitContinuation()) { - if (p > nextLine_) { + if (continuationInCharLiteral_) { + // 'a'& -> 'a''b' == "a'b" + // 'b' + if (features_.ShouldWarn( + common::LanguageFeature::MiscSourceExtensions)) { + Say(GetProvenanceRange(p, p + 1), + "Character literal continuation line should have been preceded by '&'"_port_en_US); + } + } else if (p > nextLine_) { --p; } else { insertASpace_ = true; diff --git a/flang/lib/Parser/prescan.h b/flang/lib/Parser/prescan.h index 16b2c6165f61..84e046c1b102 100644 --- a/flang/lib/Parser/prescan.h +++ b/flang/lib/Parser/prescan.h @@ -218,6 +218,7 @@ private: bool slashInCurrentStatement_{false}; bool preventHollerith_{false}; // CHARACTER*4HIMOM not Hollerith bool inCharLiteral_{false}; + bool continuationInCharLiteral_{false}; bool inPreprocessorDirective_{false}; // In some edge cases of compiler directive continuation lines, it diff --git a/flang/lib/Semantics/check-omp-structure.cpp b/flang/lib/Semantics/check-omp-structure.cpp index 2f4eb9a85427..c430375d5ed0 100644 --- a/flang/lib/Semantics/check-omp-structure.cpp +++ b/flang/lib/Semantics/check-omp-structure.cpp @@ -2996,11 +2996,14 @@ void OmpStructureChecker::Enter(const parser::OmpClause::IsDevicePtr &x) { source.ToString()); } else if (!(IsDummy(*symbol))) { context_.Say(itr->second->source, - "Variable '%s' in IS_DEVICE_PTR clause must be a dummy argument"_err_en_US, + "Variable '%s' in IS_DEVICE_PTR clause must be a dummy argument. " + "This semantic check is deprecated from OpenMP 5.2 and later."_warn_en_US, source.ToString()); } else if (IsAllocatableOrPointer(*symbol) || IsValue(*symbol)) { context_.Say(itr->second->source, - "Variable '%s' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute."_err_en_US, + "Variable '%s' in IS_DEVICE_PTR clause must be a dummy argument " + "that does not have the ALLOCATABLE, POINTER or VALUE attribute. " + "This semantic check is deprecated from OpenMP 5.2 and later."_warn_en_US, source.ToString()); } } diff --git a/flang/lib/Semantics/definable.cpp b/flang/lib/Semantics/definable.cpp index d5ffcabc7233..b73290109248 100644 --- a/flang/lib/Semantics/definable.cpp +++ b/flang/lib/Semantics/definable.cpp @@ -89,7 +89,8 @@ static const Symbol &GetRelevantSymbol(const evaluate::DataRef &dataRef, // Check the leftmost (or only) symbol from a data-ref or expression. static std::optional WhyNotDefinableBase(parser::CharBlock at, - const Scope &scope, DefinabilityFlags flags, const Symbol &original) { + const Scope &scope, DefinabilityFlags flags, const Symbol &original, + bool isWholeSymbol) { const Symbol &ultimate{original.GetUltimate()}; bool isPointerDefinition{flags.test(DefinabilityFlag::PointerDefinition)}; bool acceptAllocatable{flags.test(DefinabilityFlag::AcceptAllocatable)}; @@ -104,7 +105,8 @@ static std::optional WhyNotDefinableBase(parser::CharBlock at, } else if (auto dataRef{evaluate::ExtractDataRef( *association->expr(), true, true)}) { return WhyNotDefinableBase(at, scope, flags, - GetRelevantSymbol(*dataRef, isPointerDefinition, acceptAllocatable)); + GetRelevantSymbol(*dataRef, isPointerDefinition, acceptAllocatable), + isWholeSymbol); } } if (isTargetDefinition) { @@ -112,7 +114,8 @@ static std::optional WhyNotDefinableBase(parser::CharBlock at, return BlameSymbol(at, "'%s' is not a variable"_en_US, original); } else if (IsProtected(ultimate) && IsUseAssociated(original, scope)) { return BlameSymbol(at, "'%s' is protected in this scope"_en_US, original); - } else if (IsIntentIn(ultimate)) { + } else if (IsIntentIn(ultimate) && + (!IsPointer(ultimate) || (isWholeSymbol && isPointerDefinition))) { return BlameSymbol( at, "'%s' is an INTENT(IN) dummy argument"_en_US, original); } @@ -165,6 +168,12 @@ static std::optional WhyNotDefinableBase(parser::CharBlock at, static std::optional WhyNotDefinableLast(parser::CharBlock at, const Scope &scope, DefinabilityFlags flags, const Symbol &original) { const Symbol &ultimate{original.GetUltimate()}; + if (const auto *association{ultimate.detailsIf()}) { + if (auto dataRef{ + evaluate::ExtractDataRef(*association->expr(), true, true)}) { + return WhyNotDefinableLast(at, scope, flags, dataRef->GetLastSymbol()); + } + } if (flags.test(DefinabilityFlag::PointerDefinition)) { if (flags.test(DefinabilityFlag::AcceptAllocatable)) { if (!IsAllocatableOrObjectPointer(&ultimate)) { @@ -216,7 +225,8 @@ static std::optional WhyNotDefinable(parser::CharBlock at, const Symbol &base{GetRelevantSymbol(dataRef, flags.test(DefinabilityFlag::PointerDefinition), flags.test(DefinabilityFlag::AcceptAllocatable))}; - if (auto whyNot{WhyNotDefinableBase(at, scope, flags, base)}) { + if (auto whyNot{WhyNotDefinableBase(at, scope, flags, base, + std::holds_alternative(dataRef.u))}) { return whyNot; } else { return WhyNotDefinableLast(at, scope, flags, dataRef.GetLastSymbol()); @@ -231,12 +241,13 @@ static std::optional WhyNotDefinable(parser::CharBlock at, const Symbol &base{GetRelevantSymbol(dataRef, false, false)}; DefinabilityFlags baseFlags{flags}; baseFlags.reset(DefinabilityFlag::PointerDefinition); - return WhyNotDefinableBase(at, scope, baseFlags, base); + return WhyNotDefinableBase(at, scope, baseFlags, base, + std::holds_alternative(dataRef.u)); } std::optional WhyNotDefinable(parser::CharBlock at, const Scope &scope, DefinabilityFlags flags, const Symbol &original) { - if (auto base{WhyNotDefinableBase(at, scope, flags, original)}) { + if (auto base{WhyNotDefinableBase(at, scope, flags, original, true)}) { return base; } return WhyNotDefinableLast(at, scope, flags, original); diff --git a/flang/runtime/character.cpp b/flang/runtime/character.cpp index 571f64ec42f4..2afde7cd5e83 100644 --- a/flang/runtime/character.cpp +++ b/flang/runtime/character.cpp @@ -629,148 +629,6 @@ void RTNAME(CharacterConcatenateScalar1)( FreeMemory(old); } -void RTNAME(CharacterAssign)(Descriptor &lhs, const Descriptor &rhs, - const char *sourceFile, int sourceLine) { - Terminator terminator{sourceFile, sourceLine}; - int rank{lhs.rank()}; - RUNTIME_CHECK(terminator, rhs.rank() == 0 || rhs.rank() == rank); - SubscriptValue ub[maxRank], lhsAt[maxRank], rhsAt[maxRank]; - SubscriptValue elements{1}; - std::size_t lhsBytes{lhs.ElementBytes()}; - std::size_t rhsBytes{rhs.ElementBytes()}; - bool reallocate{lhs.IsAllocatable() && - (lhs.raw().base_addr == nullptr || lhsBytes != rhsBytes)}; - for (int j{0}; j < rank; ++j) { - lhsAt[j] = lhs.GetDimension(j).LowerBound(); - if (rhs.rank() > 0) { - SubscriptValue lhsExt{lhs.GetDimension(j).Extent()}; - SubscriptValue rhsExt{rhs.GetDimension(j).Extent()}; - ub[j] = lhsAt[j] + rhsExt - 1; - if (lhsExt != rhsExt) { - if (lhs.IsAllocatable()) { - reallocate = true; - } else { - terminator.Crash("Character array assignment: operands are not " - "conforming on dimension %d (%jd != %jd)", - j + 1, static_cast(lhsExt), - static_cast(rhsExt)); - } - } - rhsAt[j] = rhs.GetDimension(j).LowerBound(); - } else { - ub[j] = lhs.GetDimension(j).UpperBound(); - } - elements *= ub[j] - lhsAt[j] + 1; - } - void *old{nullptr}; - if (reallocate) { - old = lhs.raw().base_addr; - lhs.set_base_addr(nullptr); - lhs.raw().elem_len = lhsBytes = rhsBytes; - if (rhs.rank() > 0) { - // When the RHS is not scalar, the LHS acquires its bounds. - for (int j{0}; j < rank; ++j) { - lhsAt[j] = rhsAt[j]; - ub[j] = rhs.GetDimension(j).UpperBound(); - lhs.GetDimension(j).SetBounds(lhsAt[j], ub[j]); - } - } - RUNTIME_CHECK(terminator, lhs.Allocate() == CFI_SUCCESS); - } - switch (lhs.raw().type) { - case CFI_type_char: - switch (rhs.raw().type) { - case CFI_type_char: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), lhsBytes, - rhsBytes); - } - break; - case CFI_type_char16_t: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes, rhsBytes >> 1); - } - break; - case CFI_type_char32_t: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes, rhsBytes >> 2); - } - break; - default: - terminator.Crash( - "RHS of character assignment does not have a character type"); - } - break; - case CFI_type_char16_t: - switch (rhs.raw().type) { - case CFI_type_char: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes >> 1, rhsBytes); - } - break; - case CFI_type_char16_t: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes >> 1, rhsBytes >> 1); - } - break; - case CFI_type_char32_t: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes >> 1, rhsBytes >> 2); - } - break; - default: - terminator.Crash( - "RHS of character assignment does not have a character type"); - } - break; - case CFI_type_char32_t: - switch (rhs.raw().type) { - case CFI_type_char: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes >> 2, rhsBytes); - } - break; - case CFI_type_char16_t: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes >> 2, rhsBytes >> 1); - } - break; - case CFI_type_char32_t: - for (; elements-- > 0; - lhs.IncrementSubscripts(lhsAt), rhs.IncrementSubscripts(rhsAt)) { - CopyAndPad(lhs.Element(lhsAt), rhs.Element(rhsAt), - lhsBytes >> 2, rhsBytes >> 2); - } - break; - default: - terminator.Crash( - "RHS of character assignment does not have a character type"); - } - break; - default: - terminator.Crash( - "LHS of character assignment does not have a character type"); - } - if (reallocate) { - FreeMemory(old); - } -} - int RTNAME(CharacterCompareScalar)(const Descriptor &x, const Descriptor &y) { Terminator terminator{__FILE__, __LINE__}; RUNTIME_CHECK(terminator, x.rank() == 0); diff --git a/flang/runtime/descriptor-io.cpp b/flang/runtime/descriptor-io.cpp index 563a69e999d5..6041104773cc 100644 --- a/flang/runtime/descriptor-io.cpp +++ b/flang/runtime/descriptor-io.cpp @@ -14,7 +14,8 @@ namespace Fortran::runtime::io::descr { // Defined formatted I/O (maybe) std::optional DefinedFormattedIo(IoStatementState &io, const Descriptor &descriptor, const typeInfo::DerivedType &derived, - const typeInfo::SpecialBinding &special) { + const typeInfo::SpecialBinding &special, + const SubscriptValue subscripts[]) { std::optional peek{io.GetNextDataEdit(0 /*to peek at it*/)}; if (peek && (peek->descriptor == DataEdit::DefinedDerivedType || @@ -61,9 +62,6 @@ std::optional DefinedFormattedIo(IoStatementState &io, // I/O subroutine reads counts towards READ(SIZE=). startPos = io.InquirePos(); } - std::size_t numElements{descriptor.Elements()}; - SubscriptValue subscripts[maxRank]; - descriptor.GetLowerBounds(subscripts); if (special.IsArgDescriptor(0)) { // "dtv" argument is "class(t)", pass a descriptor auto *p{special.GetProc DefinedFormattedIo(IoStatementState &io, Descriptor &elementDesc{elementStatDesc.descriptor()}; elementDesc.Establish( derived, nullptr, 0, nullptr, CFI_attribute_pointer); - for (; numElements-- > 0; descriptor.IncrementSubscripts(subscripts)) { - elementDesc.set_base_addr(descriptor.Element(subscripts)); - p(elementDesc, unit, ioType, vListDesc, ioStat, ioMsg, ioTypeLen, - sizeof ioMsg); - if (ioStat != IostatOk) { - break; - } - } + elementDesc.set_base_addr(descriptor.Element(subscripts)); + p(elementDesc, unit, ioType, vListDesc, ioStat, ioMsg, ioTypeLen, + sizeof ioMsg); } else { // "dtv" argument is "type(t)", pass a raw pointer auto *p{special.GetProc()}; - for (; numElements-- > 0; descriptor.IncrementSubscripts(subscripts)) { - p(descriptor.Element(subscripts), unit, ioType, vListDesc, ioStat, - ioMsg, ioTypeLen, sizeof ioMsg); - if (ioStat != IostatOk) { - break; - } - } + p(descriptor.Element(subscripts), unit, ioType, vListDesc, ioStat, + ioMsg, ioTypeLen, sizeof ioMsg); } handler.Forward(ioStat, ioMsg, sizeof ioMsg); external->PopChildIo(child); @@ -119,7 +107,10 @@ bool DefinedUnformattedIo(IoStatementState &io, const Descriptor &descriptor, // Unformatted I/O must have an external unit (or child thereof). IoErrorHandler &handler{io.GetIoErrorHandler()}; ExternalFileUnit *external{io.GetExternalFileUnit()}; - RUNTIME_CHECK(handler, external != nullptr); + if (!external) { // INQUIRE(IOLENGTH=) + handler.SignalError(IostatNonExternalDefinedUnformattedIo); + return false; + } ChildIo &child{external->PushChildIo(io)}; int unit{external->unitNumber()}; int ioStat{IostatOk}; diff --git a/flang/runtime/descriptor-io.h b/flang/runtime/descriptor-io.h index 2b5bf8248aca..394578796faa 100644 --- a/flang/runtime/descriptor-io.h +++ b/flang/runtime/descriptor-io.h @@ -268,7 +268,33 @@ static bool DefaultComponentIO(IoStatementState &io, } template -static bool DefaultComponentwiseIO(IoStatementState &io, +static bool DefaultComponentwiseFormattedIO(IoStatementState &io, + const Descriptor &descriptor, const typeInfo::DerivedType &type, + const NonTbpDefinedIoTable *table, const SubscriptValue subscripts[]) { + IoErrorHandler &handler{io.GetIoErrorHandler()}; + const Descriptor &compArray{type.component()}; + RUNTIME_CHECK(handler, compArray.rank() == 1); + std::size_t numComponents{compArray.Elements()}; + SubscriptValue at[maxRank]; + compArray.GetLowerBounds(at); + for (std::size_t k{0}; k < numComponents; + ++k, compArray.IncrementSubscripts(at)) { + const typeInfo::Component &component{ + *compArray.Element(at)}; + if (!DefaultComponentIO( + io, component, descriptor, subscripts, handler, table)) { + // Return true for NAMELIST input if any component appeared. + auto *listInput{ + io.get_if>()}; + return DIR == Direction::Input && k > 0 && listInput && + listInput->inNamelistSequence(); + } + } + return true; +} + +template +static bool DefaultComponentwiseUnformattedIO(IoStatementState &io, const Descriptor &descriptor, const typeInfo::DerivedType &type, const NonTbpDefinedIoTable *table) { IoErrorHandler &handler{io.GetIoErrorHandler()}; @@ -288,11 +314,7 @@ static bool DefaultComponentwiseIO(IoStatementState &io, *compArray.Element(at)}; if (!DefaultComponentIO( io, component, descriptor, subscripts, handler, table)) { - // Truncated nonempty namelist input sequence? - auto *listInput{ - io.get_if>()}; - return DIR == Direction::Input && (j > 0 || k > 0) && listInput && - listInput->inNamelistSequence(); + return false; } } } @@ -300,7 +322,8 @@ static bool DefaultComponentwiseIO(IoStatementState &io, } std::optional DefinedFormattedIo(IoStatementState &, const Descriptor &, - const typeInfo::DerivedType &, const typeInfo::SpecialBinding &); + const typeInfo::DerivedType &, const typeInfo::SpecialBinding &, + const SubscriptValue[]); template static bool FormattedDerivedTypeIO(IoStatementState &io, @@ -311,37 +334,54 @@ static bool FormattedDerivedTypeIO(IoStatementState &io, RUNTIME_CHECK(handler, addendum != nullptr); const typeInfo::DerivedType *type{addendum->derivedType()}; RUNTIME_CHECK(handler, type != nullptr); + std::optional nonTbpSpecial; + const typeInfo::SpecialBinding *special{nullptr}; if (table) { if (const auto *definedIo{table->Find(*type, DIR == Direction::Input ? common::DefinedIo::ReadFormatted : common::DefinedIo::WriteFormatted)}) { if (definedIo->subroutine) { - typeInfo::SpecialBinding special{DIR == Direction::Input + nonTbpSpecial.emplace(DIR == Direction::Input ? typeInfo::SpecialBinding::Which::ReadFormatted : typeInfo::SpecialBinding::Which::WriteFormatted, definedIo->subroutine, definedIo->isDtvArgPolymorphic, false, - false}; - if (std::optional wasDefined{ - DefinedFormattedIo(io, descriptor, *type, special)}) { - return *wasDefined; - } - } else { - return DefaultComponentwiseIO(io, descriptor, *type, table); + false); + special = &*nonTbpSpecial; } } } - if (const typeInfo::SpecialBinding * - special{type->FindSpecialBinding(DIR == Direction::Input - ? typeInfo::SpecialBinding::Which::ReadFormatted - : typeInfo::SpecialBinding::Which::WriteFormatted)}) { - if (!table || !table->ignoreNonTbpEntries || special->isTypeBound()) { - if (std::optional wasDefined{ - DefinedFormattedIo(io, descriptor, *type, *special)}) { - return *wasDefined; // defined I/O was applied + if (!special) { + if (const typeInfo::SpecialBinding * + binding{type->FindSpecialBinding(DIR == Direction::Input + ? typeInfo::SpecialBinding::Which::ReadFormatted + : typeInfo::SpecialBinding::Which::WriteFormatted)}) { + if (!table || !table->ignoreNonTbpEntries || binding->isTypeBound()) { + special = binding; } } } - return DefaultComponentwiseIO(io, descriptor, *type, table); + SubscriptValue subscripts[maxRank]; + descriptor.GetLowerBounds(subscripts); + std::size_t numElements{descriptor.Elements()}; + for (std::size_t j{0}; j < numElements; + ++j, descriptor.IncrementSubscripts(subscripts)) { + std::optional result; + if (special) { + result = DefinedFormattedIo(io, descriptor, *type, *special, subscripts); + } + if (!result) { + result = DefaultComponentwiseFormattedIO( + io, descriptor, *type, table, subscripts); + } + if (!result.value()) { + // Return true for NAMELIST input if we got anything. + auto *listInput{ + io.get_if>()}; + return DIR == Direction::Input && j > 0 && listInput && + listInput->inNamelistSequence(); + } + } + return true; } bool DefinedUnformattedIo(IoStatementState &, const Descriptor &, @@ -371,7 +411,8 @@ static bool UnformattedDescriptorIO(IoStatementState &io, return *wasDefined; } } else { - return DefaultComponentwiseIO(io, descriptor, *type, table); + return DefaultComponentwiseUnformattedIO( + io, descriptor, *type, table); } } } @@ -388,7 +429,7 @@ static bool UnformattedDescriptorIO(IoStatementState &io, // TODO: If no component at any level has defined READ or WRITE // (as appropriate), the elements are contiguous, and no byte swapping // is active, do a block transfer via the code below. - return DefaultComponentwiseIO(io, descriptor, *type, table); + return DefaultComponentwiseUnformattedIO(io, descriptor, *type, table); } else { // intrinsic type unformatted I/O auto *externalUnf{io.get_if>()}; diff --git a/flang/runtime/edit-input.cpp b/flang/runtime/edit-input.cpp index 4e8c9aa868a6..822099b5141b 100644 --- a/flang/runtime/edit-input.cpp +++ b/flang/runtime/edit-input.cpp @@ -64,10 +64,15 @@ static bool EditBOZInput( } // Count significant digits after any leading white space & zeroes int digits{0}; + int significantBits{0}; for (; next; next = io.NextInField(remaining, edit)) { char32_t ch{*next}; if (ch == ' ' || ch == '\t') { - continue; + if (edit.modes.editingFlags & blankZero) { + ch = '0'; // BZ mode - treat blank as if it were zero + } else { + continue; + } } if (ch >= '0' && ch <= '1') { } else if (LOG2_BASE >= 3 && ch >= '2' && ch <= '7') { @@ -79,9 +84,22 @@ static bool EditBOZInput( "Bad character '%lc' in B/O/Z input field", ch); return false; } - ++digits; + if (digits++ == 0) { + significantBits = 4; + if (ch >= '0' && ch <= '1') { + significantBits = 1; + } else if (ch >= '2' && ch <= '3') { + significantBits = 2; + } else if (ch >= '4' && ch <= '7') { + significantBits = 3; + } else { + significantBits = 4; + } + } else { + significantBits += LOG2_BASE; + } } - auto significantBytes{static_cast(digits * LOG2_BASE + 7) / 8}; + auto significantBytes{static_cast(significantBits + 7) / 8}; if (significantBytes > bytes) { io.GetIoErrorHandler().SignalError(IostatBOZInputOverflow, "B/O/Z input of %d digits overflows %zd-byte variable", digits, bytes); @@ -96,12 +114,17 @@ static bool EditBOZInput( auto *data{reinterpret_cast(n) + (isHostLittleEndian ? significantBytes - 1 : 0)}; int shift{((digits - 1) * LOG2_BASE) & 7}; - if (shift + LOG2_BASE > 8) { - shift -= 8; // misaligned octal - } while (digits > 0) { char32_t ch{*io.NextInField(remaining, edit)}; int digit{0}; + if (ch == ' ' || ch == '\t') { + if (edit.modes.editingFlags & blankZero) { + ch = '0'; // BZ mode - treat blank as if it were zero + } else { + continue; + } + } + --digits; if (ch >= '0' && ch <= '9') { digit = ch - '0'; } else if (ch >= 'A' && ch <= 'F') { @@ -111,12 +134,11 @@ static bool EditBOZInput( } else { continue; } - --digits; if (shift < 0) { - shift += 8; - if (shift + LOG2_BASE > 8) { // misaligned octal - *data |= digit >> (8 - shift); + if (shift + LOG2_BASE > 0) { // misaligned octal + *data |= digit >> -shift; } + shift += 8; data += increment; } *data |= digit << shift; @@ -894,20 +916,20 @@ static bool EditListDirectedCharacterInput( } template -bool EditCharacterInput( - IoStatementState &io, const DataEdit &edit, CHAR *x, std::size_t length) { +bool EditCharacterInput(IoStatementState &io, const DataEdit &edit, CHAR *x, + std::size_t lengthChars) { switch (edit.descriptor) { case DataEdit::ListDirected: - return EditListDirectedCharacterInput(io, x, length, edit); + return EditListDirectedCharacterInput(io, x, lengthChars, edit); case 'A': case 'G': break; case 'B': - return EditBOZInput<1>(io, edit, x, length * sizeof *x); + return EditBOZInput<1>(io, edit, x, lengthChars * sizeof *x); case 'O': - return EditBOZInput<3>(io, edit, x, length * sizeof *x); + return EditBOZInput<3>(io, edit, x, lengthChars * sizeof *x); case 'Z': - return EditBOZInput<4>(io, edit, x, length * sizeof *x); + return EditBOZInput<4>(io, edit, x, lengthChars * sizeof *x); default: io.GetIoErrorHandler().SignalError(IostatErrorInFormat, "Data edit descriptor '%c' may not be used with a CHARACTER data item", @@ -915,27 +937,31 @@ bool EditCharacterInput( return false; } const ConnectionState &connection{io.GetConnectionState()}; - std::size_t remaining{length}; + std::size_t remainingChars{lengthChars}; + // Skip leading characters. + // Their bytes don't count towards INQUIRE(IOLENGTH=). + std::size_t skipChars{0}; if (edit.width && *edit.width > 0) { - remaining = *edit.width; + remainingChars = *edit.width; + if (remainingChars > lengthChars) { + skipChars = remainingChars - lengthChars; + } } // When the field is wider than the variable, we drop the leading // characters. When the variable is wider than the field, there can be // trailing padding or an EOR condition. const char *input{nullptr}; - std::size_t ready{0}; - // Skip leading bytes. - // These bytes don't count towards INQUIRE(IOLENGTH=). - std::size_t skip{remaining > length ? remaining - length : 0}; + std::size_t readyBytes{0}; // Transfer payload bytes; these do count. - while (remaining > 0) { - if (ready == 0) { - ready = io.GetNextInputBytes(input); - if (ready == 0 || (ready < remaining && edit.modes.nonAdvancing)) { - if (io.CheckForEndOfRecord(ready)) { - if (ready == 0) { + while (remainingChars > 0) { + if (readyBytes == 0) { + readyBytes = io.GetNextInputBytes(input); + if (readyBytes == 0 || + (readyBytes < remainingChars && edit.modes.nonAdvancing)) { + if (io.CheckForEndOfRecord(readyBytes)) { + if (readyBytes == 0) { // PAD='YES' and no more data - std::fill_n(x, length, ' '); + std::fill_n(x, lengthChars, ' '); return !io.GetIoErrorHandler().InError(); } else { // Do partial read(s) then pad on last iteration @@ -945,63 +971,64 @@ bool EditCharacterInput( } } } - std::size_t chunk; - bool skipping{skip > 0}; + std::size_t chunkBytes; + std::size_t chunkChars{1}; + bool skipping{skipChars > 0}; if (connection.isUTF8) { - chunk = MeasureUTF8Bytes(*input); + chunkBytes = MeasureUTF8Bytes(*input); if (skipping) { - --skip; + --skipChars; } else if (auto ucs{DecodeUTF8(input)}) { *x++ = *ucs; - --length; - } else if (chunk == 0) { + --lengthChars; + } else if (chunkBytes == 0) { // error recovery: skip bad encoding - chunk = 1; + chunkBytes = 1; } - --remaining; } else if (connection.internalIoCharKind > 1) { // Reading from non-default character internal unit - chunk = connection.internalIoCharKind; + chunkBytes = connection.internalIoCharKind; if (skipping) { - --skip; + --skipChars; } else { char32_t buffer{0}; - std::memcpy(&buffer, input, chunk); + std::memcpy(&buffer, input, chunkBytes); *x++ = buffer; - --length; + --lengthChars; } - --remaining; } else if constexpr (sizeof *x > 1) { // Read single byte with expansion into multi-byte CHARACTER - chunk = 1; + chunkBytes = 1; if (skipping) { - --skip; + --skipChars; } else { *x++ = static_cast(*input); - --length; + --lengthChars; } - --remaining; } else { // single bytes -> default CHARACTER if (skipping) { - chunk = std::min(skip, ready); - skip -= chunk; + chunkBytes = std::min(skipChars, readyBytes); + chunkChars = chunkBytes; + skipChars -= chunkChars; } else { - chunk = std::min(remaining, ready); - std::memcpy(x, input, chunk); - x += chunk; - length -= chunk; + chunkBytes = std::min(remainingChars, readyBytes); + chunkBytes = std::min(lengthChars, chunkBytes); + chunkChars = chunkBytes; + std::memcpy(x, input, chunkBytes); + x += chunkBytes; + lengthChars -= chunkChars; } - remaining -= chunk; } - input += chunk; + input += chunkBytes; + remainingChars -= chunkChars; if (!skipping) { - io.GotChar(chunk); + io.GotChar(chunkBytes); } - io.HandleRelativePosition(chunk); - ready -= chunk; + io.HandleRelativePosition(chunkBytes); + readyBytes -= chunkBytes; } // Pad the remainder of the input variable, if any. - std::fill_n(x, length, ' '); + std::fill_n(x, lengthChars, ' '); return CheckCompleteListDirectedField(io, edit); } diff --git a/flang/runtime/edit-output.cpp b/flang/runtime/edit-output.cpp index 18b209bc6798..a4ce0b12f911 100644 --- a/flang/runtime/edit-output.cpp +++ b/flang/runtime/edit-output.cpp @@ -140,7 +140,7 @@ bool EditIntegerOutput(IoStatementState &io, const DataEdit &edit, return EditBOZOutput<4>( io, edit, reinterpret_cast(&n), KIND); case 'L': - return EditLogicalOutput(io, edit, *reinterpret_cast(&n)); + return EditLogicalOutput(io, edit, n != 0 ? true : false); case 'A': // legacy extension return EditCharacterOutput( io, edit, reinterpret_cast(&n), sizeof n); @@ -433,27 +433,28 @@ bool RealOutputEditing::EditFOutput(const DataEdit &edit) { } // Multiple conversions may be needed to get the right number of // effective rounded fractional digits. - int extraDigits{0}; bool canIncrease{true}; - while (true) { + for (int extraDigits{fracDigits == 0 ? 1 : 0};;) { decimal::ConversionToDecimalResult converted{ ConvertToDecimal(extraDigits + fracDigits, rounding, flags)}; - if (IsInfOrNaN(converted.str, static_cast(converted.length))) { + const char *convertedStr{converted.str}; + if (IsInfOrNaN(convertedStr, static_cast(converted.length))) { return editWidth > 0 && converted.length > static_cast(editWidth) ? EmitRepeated(io_, '*', editWidth) : EmitPrefix(edit, converted.length, editWidth) && - EmitAscii(io_, converted.str, converted.length) && + EmitAscii(io_, convertedStr, converted.length) && EmitSuffix(edit); } int expo{converted.decimalExponent + edit.modes.scale /*kP*/}; - int signLength{*converted.str == '-' || *converted.str == '+' ? 1 : 0}; + int signLength{*convertedStr == '-' || *convertedStr == '+' ? 1 : 0}; int convertedDigits{static_cast(converted.length) - signLength}; if (IsZero()) { // don't treat converted "0" as significant digit expo = 0; convertedDigits = 0; } - int trailingOnes{0}; + bool isNegative{*convertedStr == '-'}; + char one[2]; if (expo > extraDigits && extraDigits >= 0 && canIncrease) { extraDigits = expo; if (!edit.digits.has_value()) { // F0 @@ -462,24 +463,45 @@ bool RealOutputEditing::EditFOutput(const DataEdit &edit) { canIncrease = false; // only once continue; } else if (expo == -fracDigits && convertedDigits > 0) { - if ((rounding == decimal::FortranRounding::RoundUp && - *converted.str != '-') || - (rounding == decimal::FortranRounding::RoundDown && - *converted.str == '-') || - (rounding == decimal::FortranRounding::RoundToZero && - rounding != edit.modes.round && // it changed below - converted.str[signLength] >= '5')) { - // Round up/down to a scaled 1 + // Result will be either a signed zero or power of ten, depending + // on rounding. + char leading{convertedStr[signLength]}; + bool roundToPowerOfTen{false}; + switch (edit.modes.round) { + case decimal::FortranRounding::RoundUp: + roundToPowerOfTen = !isNegative; + break; + case decimal::FortranRounding::RoundDown: + roundToPowerOfTen = isNegative; + break; + case decimal::FortranRounding::RoundToZero: + break; + case decimal::FortranRounding::RoundNearest: + if (leading == '5' && + rounding == decimal::FortranRounding::RoundNearest) { + // Try again, rounding away from zero. + rounding = isNegative ? decimal::FortranRounding::RoundDown + : decimal::FortranRounding::RoundUp; + extraDigits = 1 - fracDigits; // just one digit needed + continue; + } + roundToPowerOfTen = leading > '5'; + break; + case decimal::FortranRounding::RoundCompatible: + roundToPowerOfTen = leading >= '5'; + break; + } + if (roundToPowerOfTen) { ++expo; - convertedDigits = 0; - trailingOnes = 1; - } else if (rounding != decimal::FortranRounding::RoundToZero) { - // Convert again with truncation so first digit can be checked - // on the next iteration by the code above - rounding = decimal::FortranRounding::RoundToZero; - continue; + convertedDigits = 1; + if (signLength > 0) { + one[0] = *convertedStr; + one[1] = '1'; + } else { + one[0] = '1'; + } + convertedStr = one; } else { - // Value rounds down to zero expo = 0; convertedDigits = 0; } @@ -493,17 +515,14 @@ bool RealOutputEditing::EditFOutput(const DataEdit &edit) { int digitsAfterPoint{convertedDigits - digitsBeforePoint}; int trailingZeroes{flags & decimal::Minimize ? 0 - : std::max(0, - fracDigits - - (zeroesAfterPoint + digitsAfterPoint + trailingOnes))}; + : std::max(0, fracDigits - (zeroesAfterPoint + digitsAfterPoint))}; if (digitsBeforePoint + zeroesBeforePoint + zeroesAfterPoint + - digitsAfterPoint + trailingOnes + trailingZeroes == + digitsAfterPoint + trailingZeroes == 0) { zeroesBeforePoint = 1; // "." -> "0." } int totalLength{signLength + digitsBeforePoint + zeroesBeforePoint + - 1 /*'.'*/ + zeroesAfterPoint + digitsAfterPoint + trailingOnes + - trailingZeroes}; + 1 /*'.'*/ + zeroesAfterPoint + digitsAfterPoint + trailingZeroes}; int width{editWidth > 0 ? editWidth : totalLength}; if (totalLength > width) { return EmitRepeated(io_, '*', width); @@ -513,13 +532,12 @@ bool RealOutputEditing::EditFOutput(const DataEdit &edit) { ++totalLength; } return EmitPrefix(edit, totalLength, width) && - EmitAscii(io_, converted.str, signLength + digitsBeforePoint) && + EmitAscii(io_, convertedStr, signLength + digitsBeforePoint) && EmitRepeated(io_, '0', zeroesBeforePoint) && EmitAscii(io_, edit.modes.editingFlags & decimalComma ? "," : ".", 1) && EmitRepeated(io_, '0', zeroesAfterPoint) && - EmitAscii(io_, converted.str + signLength + digitsBeforePoint, + EmitAscii(io_, convertedStr + signLength + digitsBeforePoint, digitsAfterPoint) && - EmitRepeated(io_, '1', trailingOnes) && EmitRepeated(io_, '0', trailingZeroes) && EmitRepeated(io_, ' ', trailingBlanks_) && EmitSuffix(edit); } diff --git a/flang/runtime/internal-unit.cpp b/flang/runtime/internal-unit.cpp index aa7130f3a6a5..e3fffaa6f378 100644 --- a/flang/runtime/internal-unit.cpp +++ b/flang/runtime/internal-unit.cpp @@ -43,11 +43,9 @@ InternalDescriptorUnit::InternalDescriptorUnit( template void InternalDescriptorUnit::EndIoStatement() { if constexpr (DIR == Direction::Output) { - // Clear the remainder of the current record if anything was written - // to it, or if it is the only record. + // Clear the remainder of the current record. auto end{endfileRecordNumber.value_or(0)}; - if (currentRecordNumber < end && - (end == 2 || furthestPositionInRecord > 0)) { + if (currentRecordNumber < end) { BlankFillOutputRecord(); } } diff --git a/flang/runtime/io-stmt.h b/flang/runtime/io-stmt.h index d4ceb8326524..91169f6c6e32 100644 --- a/flang/runtime/io-stmt.h +++ b/flang/runtime/io-stmt.h @@ -92,8 +92,8 @@ public: std::size_t GetNextInputBytes(const char *&); bool AdvanceRecord(int = 1); void BackspaceRecord(); - void HandleRelativePosition(std::int64_t); - void HandleAbsolutePosition(std::int64_t); // for r* in list I/O + void HandleRelativePosition(std::int64_t byteOffset); + void HandleAbsolutePosition(std::int64_t byteOffset); // for r* in list I/O std::optional GetNextDataEdit(int maxRepeat = 1); ExternalFileUnit *GetExternalFileUnit() const; // null if internal unit bool BeginReadingRecord(); @@ -124,7 +124,11 @@ public: // Vacant after the end of the current record std::optional GetCurrentChar(std::size_t &byteCount); - // For fixed-width fields, return the number of remaining characters. + // The "remaining" arguments to CueUpInput(), SkipSpaces(), & NextInField() + // are always in units of bytes, not characters; the distinction matters + // for internal input from CHARACTER(KIND=2 and 4). + + // For fixed-width fields, return the number of remaining bytes. // Skip over leading blanks. std::optional CueUpInput(const DataEdit &edit) { std::optional remaining; @@ -134,6 +138,10 @@ public: } else { if (edit.width.value_or(0) > 0) { remaining = *edit.width; + if (int bytesPerChar{GetConnectionState().internalIoCharKind}; + bytesPerChar > 1) { + *remaining *= bytesPerChar; + } } SkipSpaces(remaining); } diff --git a/flang/runtime/iostat.cpp b/flang/runtime/iostat.cpp index cc5641693a07..c993b778e9e1 100644 --- a/flang/runtime/iostat.cpp +++ b/flang/runtime/iostat.cpp @@ -115,6 +115,8 @@ const char *IostatErrorString(int iostat) { return "NEWUNIT= without FILE= or STATUS='SCRATCH'"; case IostatBadListDirectedInputSeparator: return "List-directed input value has trailing unused characters"; + case IostatNonExternalDefinedUnformattedIo: + return "Defined unformatted I/O without an external unit"; default: return nullptr; } diff --git a/flang/runtime/unit.cpp b/flang/runtime/unit.cpp index 5fa8565c2f61..e4f346ae941f 100644 --- a/flang/runtime/unit.cpp +++ b/flang/runtime/unit.cpp @@ -355,6 +355,7 @@ bool ExternalFileUnit::Emit(const char *data, std::size_t bytes, } positionInRecord += bytes; furthestPositionInRecord = furthestAfter; + anyWriteSinceLastPositioning_ = true; return true; } @@ -440,6 +441,14 @@ bool ExternalFileUnit::BeginReadingRecord(IoErrorHandler &handler) { RUNTIME_CHECK(handler, direction_ == Direction::Input); if (!beganReadingRecord_) { beganReadingRecord_ = true; + // Don't use IsAtEOF() to check for an EOF condition here, just detect + // it from a failed or short read from the file. IsAtEOF() could be + // wrong for formatted input if actual newline characters had been + // written in-band by previous WRITEs before a REWIND. In fact, + // now that we know that the unit is being used for input (again), + // it's best to reset endfileRecordNumber and ensure IsAtEOF() will + // now be true on return only if it gets set by HitEndOnRead(). + endfileRecordNumber.reset(); if (access == Access::Direct) { CheckDirectAccess(handler); auto need{static_cast(recordOffsetInFrame_ + *openRecl)}; @@ -451,18 +460,19 @@ bool ExternalFileUnit::BeginReadingRecord(IoErrorHandler &handler) { HitEndOnRead(handler); } } else { + if (anyWriteSinceLastPositioning_ && access == Access::Sequential) { + // Most Fortran implementations allow a READ after a WRITE; + // the read then just hits an EOF. + DoEndfile(handler); + } recordLength.reset(); - if (IsAtEOF()) { - handler.SignalEnd(); - } else { - RUNTIME_CHECK(handler, isUnformatted.has_value()); - if (*isUnformatted) { - if (access == Access::Sequential) { - BeginSequentialVariableUnformattedInputRecord(handler); - } - } else { // formatted sequential or stream - BeginVariableFormattedInputRecord(handler); + RUNTIME_CHECK(handler, isUnformatted.has_value()); + if (*isUnformatted) { + if (access == Access::Sequential) { + BeginSequentialVariableUnformattedInputRecord(handler); } + } else { // formatted sequential or stream + BeginVariableFormattedInputRecord(handler); } } } @@ -595,8 +605,8 @@ void ExternalFileUnit::BackspaceRecord(IoErrorHandler &handler) { if (IsAfterEndfile()) { // BACKSPACE after explicit ENDFILE currentRecordNumber = *endfileRecordNumber; - } else if (leftTabLimit) { - // BACKSPACE after non-advancing I/O + } else if (leftTabLimit && direction_ == Direction::Input) { + // BACKSPACE after non-advancing input leftTabLimit.reset(); } else { DoImpliedEndfile(handler); @@ -615,6 +625,7 @@ void ExternalFileUnit::BackspaceRecord(IoErrorHandler &handler) { } } BeginRecord(); + anyWriteSinceLastPositioning_ = false; } } @@ -667,6 +678,7 @@ void ExternalFileUnit::Rewind(IoErrorHandler &handler) { SetPosition(0, handler); currentRecordNumber = 1; leftTabLimit.reset(); + anyWriteSinceLastPositioning_ = false; } } @@ -727,6 +739,7 @@ void ExternalFileUnit::EndIoStatement() { void ExternalFileUnit::BeginSequentialVariableUnformattedInputRecord( IoErrorHandler &handler) { + RUNTIME_CHECK(handler, access == Access::Sequential); std::int32_t header{0}, footer{0}; std::size_t need{recordOffsetInFrame_ + sizeof header}; std::size_t got{ReadFrame(frameOffsetInFile_, need, handler)}; @@ -896,28 +909,29 @@ void ExternalFileUnit::BackspaceVariableFormattedRecord( } void ExternalFileUnit::DoImpliedEndfile(IoErrorHandler &handler) { - if (!impliedEndfile_ && direction_ == Direction::Output && IsRecordFile() && - access != Access::Direct && leftTabLimit) { - // Complete partial record after non-advancing write before - // positioning or closing the unit. Usually sets impliedEndfile_. - AdvanceRecord(handler); - } - if (impliedEndfile_) { - impliedEndfile_ = false; - if (access != Access::Direct && IsRecordFile() && mayPosition()) { + if (access != Access::Direct) { + if (!impliedEndfile_ && leftTabLimit && direction_ == Direction::Output) { + // Flush a partial record after non-advancing output + impliedEndfile_ = true; + } + if (impliedEndfile_ && mayPosition()) { DoEndfile(handler); } } + impliedEndfile_ = false; } void ExternalFileUnit::DoEndfile(IoErrorHandler &handler) { if (IsRecordFile() && access != Access::Direct) { furthestPositionInRecord = std::max(positionInRecord, furthestPositionInRecord); - if (leftTabLimit) { - // Last read/write was non-advancing, so AdvanceRecord() was not called. - leftTabLimit.reset(); - ++currentRecordNumber; + if (leftTabLimit) { // last I/O was non-advancing + if (access == Access::Sequential && direction_ == Direction::Output) { + AdvanceRecord(handler); + } else { // Access::Stream or input + leftTabLimit.reset(); + ++currentRecordNumber; + } } endfileRecordNumber = currentRecordNumber; } @@ -928,6 +942,7 @@ void ExternalFileUnit::DoEndfile(IoErrorHandler &handler) { TruncateFrame(frameOffsetInFile_, handler); BeginRecord(); impliedEndfile_ = false; + anyWriteSinceLastPositioning_ = false; } void ExternalFileUnit::CommitWrites() { diff --git a/flang/runtime/unit.h b/flang/runtime/unit.h index 1ec3013ba82b..140fda3c4d2a 100644 --- a/flang/runtime/unit.h +++ b/flang/runtime/unit.h @@ -140,6 +140,7 @@ private: Direction direction_{Direction::Output}; bool impliedEndfile_{false}; // sequential/stream output has taken place bool beganReadingRecord_{false}; + bool anyWriteSinceLastPositioning_{false}; bool directAccessRecWasSet_{false}; // REC= appeared // Subtle: The beginning of the frame can't be allowed to advance // during a single list-directed READ due to the possibility of a diff --git a/flang/test/Driver/driver-help-hidden.f90 b/flang/test/Driver/driver-help-hidden.f90 index 8cb8b54d5941..9a11a7a571ff 100644 --- a/flang/test/Driver/driver-help-hidden.f90 +++ b/flang/test/Driver/driver-help-hidden.f90 @@ -64,6 +64,7 @@ ! CHECK-NEXT: Select Windows run-time library ! CHECK-NEXT: -fno-automatic Implies the SAVE attribute for non-automatic local objects in subprograms unless RECURSIVE ! CHECK-NEXT: -fno-color-diagnostics Disable colors in diagnostics +! CHECK-NEXT: -fno-fortran-main Do not include Fortran_main.a (provided by Flang) when linking ! CHECK-NEXT: -fno-integrated-as Disable the integrated assembler ! CHECK-NEXT: -fno-lto Disable LTO mode (default) ! CHECK-NEXT: -fno-ppc-native-vector-element-order diff --git a/flang/test/Driver/driver-help.f90 b/flang/test/Driver/driver-help.f90 index 0607ffde2378..e0e74dc56f33 100644 --- a/flang/test/Driver/driver-help.f90 +++ b/flang/test/Driver/driver-help.f90 @@ -54,6 +54,7 @@ ! HELP-NEXT: Select Windows run-time library ! HELP-NEXT: -fno-automatic Implies the SAVE attribute for non-automatic local objects in subprograms unless RECURSIVE ! HELP-NEXT: -fno-color-diagnostics Disable colors in diagnostics +! HELP-NEXT: -fno-fortran-main Do not include Fortran_main.a (provided by Flang) when linking ! HELP-NEXT: -fno-integrated-as Disable the integrated assembler ! HELP-NEXT: -fno-lto Disable LTO mode (default) ! HELP-NEXT: -fno-ppc-native-vector-element-order diff --git a/flang/test/Driver/no-duplicate-main.f90 b/flang/test/Driver/no-duplicate-main.f90 index 4e33f4f2aeba..12d5e46247ba 100644 --- a/flang/test/Driver/no-duplicate-main.f90 +++ b/flang/test/Driver/no-duplicate-main.f90 @@ -4,6 +4,8 @@ ! RUN: %flang -o %t -c %s ! RUN: not %flang -o %t.exe %t %t.c-object 2>&1 +! RUN: %flang -fno-fortran-main -o %t.exe %t %t.c-object 2>&1 + ! TODO: potentially add further checks to ensure that proper ! linker error messages are detected and checked via ! FileCheck. diff --git a/flang/test/Fir/array-coor.fir b/flang/test/Fir/array-coor.fir index 738acd7dd91f..a765670d20b2 100644 --- a/flang/test/Fir/array-coor.fir +++ b/flang/test/Fir/array-coor.fir @@ -9,12 +9,12 @@ func.func @array_coor_box_value(%29 : !fir.box>, } // CHECK-LABEL: define double @array_coor_box_value -// CHECK: %[[t3:.*]] = sub i64 %{{.*}}, 1 -// CHECK: %[[t4:.*]] = mul i64 %[[t3]], 1 +// CHECK: %[[t3:.*]] = sub nsw i64 %{{.*}}, 1 +// CHECK: %[[t4:.*]] = mul nsw i64 %[[t3]], 1 // CHECK: %[[t5:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 7, i32 0, i32 2 // CHECK: %[[t6:.*]] = load i64, ptr %[[t5]] -// CHECK: %[[t7:.*]] = mul i64 %[[t4]], %[[t6]] -// CHECK: %[[t8:.*]] = add i64 %[[t7]], 0 +// CHECK: %[[t7:.*]] = mul nsw i64 %[[t4]], %[[t6]] +// CHECK: %[[t8:.*]] = add nsw i64 %[[t7]], 0 // CHECK: %[[t9:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %{{.*}}, i32 0, i32 0 // CHECK: %[[t10:.*]] = load ptr, ptr %[[t9]] // CHECK: %[[t11:.*]] = getelementptr i8, ptr %[[t10]], i64 %[[t8]] @@ -36,8 +36,8 @@ func.func private @take_int(%arg0: !fir.ref) -> () // CHECK-SAME: ptr %[[VAL_0:.*]]) // CHECK: %[[VAL_1:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %[[VAL_0]], i32 0, i32 7, i32 0, i32 2 // CHECK: %[[VAL_2:.*]] = load i64, ptr %[[VAL_1]] -// CHECK: %[[VAL_3:.*]] = mul i64 1, %[[VAL_2]] -// CHECK: %[[VAL_4:.*]] = add i64 %[[VAL_3]], 0 +// CHECK: %[[VAL_3:.*]] = mul nsw i64 1, %[[VAL_2]] +// CHECK: %[[VAL_4:.*]] = add nsw i64 %[[VAL_3]], 0 // CHECK: %[[VAL_5:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]], ptr, [1 x i64] }, ptr %[[VAL_0]], i32 0, i32 0 // CHECK: %[[VAL_6:.*]] = load ptr, ptr %[[VAL_5]] // CHECK: %[[VAL_7:.*]] = getelementptr i8, ptr %[[VAL_6]], i64 %[[VAL_4]] diff --git a/flang/test/Fir/arrexp.fir b/flang/test/Fir/arrexp.fir index 87a276360825..5d265a5e3a08 100644 --- a/flang/test/Fir/arrexp.fir +++ b/flang/test/Fir/arrexp.fir @@ -114,8 +114,8 @@ func.func @f5(%arg0: !fir.box>, %arg1: !fir.box (!fir.array) { // CHECK: %[[B_STRIDE_GEP:.*]] = getelementptr {{.*}}, ptr %[[B]], i32 0, i32 7, i32 0, i32 2 // CHECK: %[[B_STRIDE:.*]] = load i64, ptr %[[B_STRIDE_GEP]] - // CHECK: %[[B_DIM_OFFSET:.*]] = mul i64 %{{.*}}, %[[B_STRIDE]] - // CHECK: %[[B_OFFSET:.*]] = add i64 %[[B_DIM_OFFSET]], 0 + // CHECK: %[[B_DIM_OFFSET:.*]] = mul nsw i64 %{{.*}}, %[[B_STRIDE]] + // CHECK: %[[B_OFFSET:.*]] = add nsw i64 %[[B_DIM_OFFSET]], 0 // CHECK: %[[B_BASE_GEP:.*]] = getelementptr {{.*}}, ptr %{{.*}}, i32 0, i32 0 // CHECK: %[[B_BASE:.*]] = load ptr, ptr %[[B_BASE_GEP]] // CHECK: %[[B_VOID_ADDR:.*]] = getelementptr i8, ptr %[[B_BASE]], i64 %[[B_OFFSET]] @@ -172,7 +172,7 @@ func.func @f7(%arg0: !fir.ref, %arg1: !fir.box>) { %0 = fir.shift %c4 : (index) -> !fir.shift<1> // CHECK: %[[STRIDE_GEP:.*]] = getelementptr {{.*}}, ptr %[[Y]], i32 0, i32 7, i32 0, i32 2 // CHECK: %[[STRIDE:.*]] = load i64, ptr %[[STRIDE_GEP]] - // CHECK: mul i64 96, %[[STRIDE]] + // CHECK: mul nsw i64 96, %[[STRIDE]] %1 = fir.array_coor %arg1(%0) %c100 : (!fir.box>, !fir.shift<1>, index) -> !fir.ref %2 = fir.load %1 : !fir.ref fir.store %2 to %arg0 : !fir.ref @@ -202,7 +202,7 @@ func.func @f8(%a : !fir.ref>>, %i : i32) { func.func @f9(%i: i32, %e : i64, %j: i64, %c: !fir.ref>>) -> !fir.ref> { %s = fir.shape %e, %e : (i64, i64) -> !fir.shape<2> // CHECK: %[[CAST:.*]] = sext i32 %[[I]] to i64 - // CHECK: %[[OFFSET:.*]] = mul i64 %{{.*}}, %[[CAST]] + // CHECK: %[[OFFSET:.*]] = mul nsw i64 %{{.*}}, %[[CAST]] // CHECK: getelementptr i8, ptr %[[C]], i64 %[[OFFSET]] %a = fir.array_coor %c(%s) %j, %j typeparams %i : (!fir.ref>>, !fir.shape<2>, i64, i64, i32) -> !fir.ref> return %a : !fir.ref> diff --git a/flang/test/Fir/convert-to-llvm.fir b/flang/test/Fir/convert-to-llvm.fir index 993058ebb0a4..be82ffab7e33 100644 --- a/flang/test/Fir/convert-to-llvm.fir +++ b/flang/test/Fir/convert-to-llvm.fir @@ -2027,10 +2027,10 @@ func.func @ext_array_coor0(%arg0: !fir.ref>) { // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[C0_1:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C1]] : i64 -// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] : i64 -// CHECK: %[[SC:.*]] = llvm.mul %[[DIFF0]], %[[C1]] : i64 -// CHECK: %[[OFFSET:.*]] = llvm.add %[[SC]], %[[C0_1]] : i64 +// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C1]] overflow : i64 +// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] overflow : i64 +// CHECK: %[[SC:.*]] = llvm.mul %[[DIFF0]], %[[C1]] overflow : i64 +// CHECK: %[[OFFSET:.*]] = llvm.add %[[SC]], %[[C0_1]] overflow : i64 // CHECK: %{{.*}} = llvm.getelementptr %[[ARG0]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 // Conversion with shift and slice. @@ -2046,12 +2046,12 @@ func.func @ext_array_coor1(%arg0: !fir.ref>) { // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[C0_1:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C0]] : i64 -// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C0]] : i64 -// CHECK: %[[ADJ:.*]] = llvm.sub %[[C0]], %[[C0]] : i64 -// CHECK: %[[DIFF1:.*]] = llvm.add %[[DIFF0]], %[[ADJ]] : i64 -// CHECK: %[[STRIDE:.*]] = llvm.mul %[[DIFF1]], %[[C1]] : i64 -// CHECK: %[[OFFSET:.*]] = llvm.add %[[STRIDE]], %[[C0_1]] : i64 +// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C0]] overflow : i64 +// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C0]] overflow : i64 +// CHECK: %[[ADJ:.*]] = llvm.sub %[[C0]], %[[C0]] overflow : i64 +// CHECK: %[[DIFF1:.*]] = llvm.add %[[DIFF0]], %[[ADJ]] overflow : i64 +// CHECK: %[[STRIDE:.*]] = llvm.mul %[[DIFF1]], %[[C1]] overflow : i64 +// CHECK: %[[OFFSET:.*]] = llvm.add %[[STRIDE]], %[[C0_1]] overflow : i64 // CHECK: %{{.*}} = llvm.getelementptr %[[ARG0]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 // Conversion for a dynamic length char. @@ -2067,10 +2067,10 @@ func.func @ext_array_coor2(%arg0: !fir.ref>>) { // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[C0_1:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C1]] : i64 -// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] : i64 -// CHECK: %[[SC:.*]] = llvm.mul %[[DIFF0]], %[[C1]] : i64 -// CHECK: %[[OFFSET:.*]] = llvm.add %[[SC]], %[[C0_1]] : i64 +// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C1]] overflow : i64 +// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] overflow : i64 +// CHECK: %[[SC:.*]] = llvm.mul %[[DIFF0]], %[[C1]] overflow : i64 +// CHECK: %[[OFFSET:.*]] = llvm.add %[[SC]], %[[C0_1]] overflow : i64 // CHECK: %{{.*}} = llvm.getelementptr %[[ARG0]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 // Conversion for a `fir.box`. @@ -2086,12 +2086,12 @@ func.func @ext_array_coor3(%arg0: !fir.box>) { // CHECK: %[[C0:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[C0_1:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C1]] : i64 -// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] : i64 +// CHECK: %[[IDX:.*]] = llvm.sub %[[C0]], %[[C1]] overflow : i64 +// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] overflow : i64 // CHECK: %[[GEPSTRIDE:.*]] = llvm.getelementptr %[[ARG0]][0, 7, 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)> // CHECK: %[[LOADEDSTRIDE:.*]] = llvm.load %[[GEPSTRIDE]] : !llvm.ptr -> i64 -// CHECK: %[[SC:.*]] = llvm.mul %[[DIFF0]], %[[LOADEDSTRIDE]] : i64 -// CHECK: %[[OFFSET:.*]] = llvm.add %[[SC]], %[[C0_1]] : i64 +// CHECK: %[[SC:.*]] = llvm.mul %[[DIFF0]], %[[LOADEDSTRIDE]] overflow : i64 +// CHECK: %[[OFFSET:.*]] = llvm.add %[[SC]], %[[C0_1]] overflow : i64 // CHECK: %[[GEPADDR:.*]] = llvm.getelementptr %[[ARG0]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)> // CHECK: %[[LOADEDADDR:.*]] = llvm.load %[[GEPADDR]] : !llvm.ptr -> !llvm.ptr // CHECK: %[[GEPADDROFFSET:.*]] = llvm.getelementptr %[[LOADEDADDR]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 @@ -2115,12 +2115,12 @@ func.func @ext_array_coor4(%arg0: !fir.ref>) { // CHECK: %[[C1:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[C1_1:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[C0_1:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[IDX:.*]] = llvm.sub %[[C1]], %[[C0]] : i64 -// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] : i64 -// CHECK: %[[ADJ:.*]] = llvm.sub %[[C10]], %[[C0]] : i64 -// CHECK: %[[DIFF1:.*]] = llvm.add %[[DIFF0]], %[[ADJ]] : i64 -// CHECK: %[[STRIDE:.*]] = llvm.mul %[[DIFF1]], %[[C1_1]] : i64 -// CHECK: %[[OFFSET:.*]] = llvm.add %[[STRIDE]], %[[C0_1]] : i64 +// CHECK: %[[IDX:.*]] = llvm.sub %[[C1]], %[[C0]] overflow : i64 +// CHECK: %[[DIFF0:.*]] = llvm.mul %[[IDX]], %[[C1]] overflow : i64 +// CHECK: %[[ADJ:.*]] = llvm.sub %[[C10]], %[[C0]] overflow : i64 +// CHECK: %[[DIFF1:.*]] = llvm.add %[[DIFF0]], %[[ADJ]] overflow : i64 +// CHECK: %[[STRIDE:.*]] = llvm.mul %[[DIFF1]], %[[C1_1]] overflow : i64 +// CHECK: %[[OFFSET:.*]] = llvm.add %[[STRIDE]], %[[C0_1]] overflow : i64 // CHECK: %{{.*}} = llvm.getelementptr %[[ARG0]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 // Conversion with index type shape and slice @@ -2134,13 +2134,13 @@ func.func @ext_array_coor5(%arg0: !fir.ref>, %idx1 : index, %i // CHECK-SAME: %[[VAL_0:.*]]: !llvm.ptr, %[[VAL_1:.*]]: i64, %[[VAL_2:.*]]: i64, %[[VAL_3:.*]]: i64, %[[VAL_4:.*]]: i64, %[[VAL_5:.*]]: i64) { // CHECK: %[[VAL_6:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[VAL_7:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[VAL_8:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_9:.*]] = llvm.mul %[[VAL_8]], %[[VAL_4]] : i64 -// CHECK: %[[VAL_10:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_11:.*]] = llvm.add %[[VAL_9]], %[[VAL_10]] : i64 -// CHECK: %[[VAL_12:.*]] = llvm.mul %[[VAL_11]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_13:.*]] = llvm.add %[[VAL_12]], %[[VAL_7]] : i64 -// CHECK: %[[VAL_14:.*]] = llvm.mul %[[VAL_6]], %[[VAL_1]] : i64 +// CHECK: %[[VAL_8:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_9:.*]] = llvm.mul %[[VAL_8]], %[[VAL_4]] overflow : i64 +// CHECK: %[[VAL_10:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_11:.*]] = llvm.add %[[VAL_9]], %[[VAL_10]] overflow : i64 +// CHECK: %[[VAL_12:.*]] = llvm.mul %[[VAL_11]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_13:.*]] = llvm.add %[[VAL_12]], %[[VAL_7]] overflow : i64 +// CHECK: %[[VAL_14:.*]] = llvm.mul %[[VAL_6]], %[[VAL_1]] overflow : i64 // CHECK: %[[VAL_16:.*]] = llvm.getelementptr %[[VAL_0]][%[[VAL_13]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 // CHECK: } @@ -2155,27 +2155,27 @@ func.func @ext_array_coor6(%arg0: !fir.ref>, %idx1 : index // CHECK-SAME: %[[VAL_0:.*]]: !llvm.ptr, %[[VAL_1:.*]]: i64, %[[VAL_2:.*]]: i64, %[[VAL_3:.*]]: i64, %[[VAL_4:.*]]: i64, %[[VAL_5:.*]]: i64) { // CHECK: %[[VAL_6:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[VAL_7:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[VAL_8:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_9:.*]] = llvm.mul %[[VAL_8]], %[[VAL_4]] : i64 -// CHECK: %[[VAL_10:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_11:.*]] = llvm.add %[[VAL_9]], %[[VAL_10]] : i64 -// CHECK: %[[VAL_12:.*]] = llvm.mul %[[VAL_11]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_13:.*]] = llvm.add %[[VAL_12]], %[[VAL_7]] : i64 -// CHECK: %[[VAL_14:.*]] = llvm.mul %[[VAL_6]], %[[VAL_1]] : i64 -// CHECK: %[[VAL_15:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_16:.*]] = llvm.mul %[[VAL_15]], %[[VAL_4]] : i64 -// CHECK: %[[VAL_17:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_18:.*]] = llvm.add %[[VAL_16]], %[[VAL_17]] : i64 -// CHECK: %[[VAL_19:.*]] = llvm.mul %[[VAL_18]], %[[VAL_14]] : i64 -// CHECK: %[[VAL_20:.*]] = llvm.add %[[VAL_19]], %[[VAL_13]] : i64 -// CHECK: %[[VAL_21:.*]] = llvm.mul %[[VAL_14]], %[[VAL_1]] : i64 -// CHECK: %[[VAL_22:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_23:.*]] = llvm.mul %[[VAL_22]], %[[VAL_4]] : i64 -// CHECK: %[[VAL_24:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] : i64 -// CHECK: %[[VAL_25:.*]] = llvm.add %[[VAL_23]], %[[VAL_24]] : i64 -// CHECK: %[[VAL_26:.*]] = llvm.mul %[[VAL_25]], %[[VAL_21]] : i64 -// CHECK: %[[VAL_27:.*]] = llvm.add %[[VAL_26]], %[[VAL_20]] : i64 -// CHECK: %[[VAL_28:.*]] = llvm.mul %[[VAL_21]], %[[VAL_1]] : i64 +// CHECK: %[[VAL_8:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_9:.*]] = llvm.mul %[[VAL_8]], %[[VAL_4]] overflow : i64 +// CHECK: %[[VAL_10:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_11:.*]] = llvm.add %[[VAL_9]], %[[VAL_10]] overflow : i64 +// CHECK: %[[VAL_12:.*]] = llvm.mul %[[VAL_11]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_13:.*]] = llvm.add %[[VAL_12]], %[[VAL_7]] overflow : i64 +// CHECK: %[[VAL_14:.*]] = llvm.mul %[[VAL_6]], %[[VAL_1]] overflow : i64 +// CHECK: %[[VAL_15:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_16:.*]] = llvm.mul %[[VAL_15]], %[[VAL_4]] overflow : i64 +// CHECK: %[[VAL_17:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_18:.*]] = llvm.add %[[VAL_16]], %[[VAL_17]] overflow : i64 +// CHECK: %[[VAL_19:.*]] = llvm.mul %[[VAL_18]], %[[VAL_14]] overflow : i64 +// CHECK: %[[VAL_20:.*]] = llvm.add %[[VAL_19]], %[[VAL_13]] overflow : i64 +// CHECK: %[[VAL_21:.*]] = llvm.mul %[[VAL_14]], %[[VAL_1]] overflow : i64 +// CHECK: %[[VAL_22:.*]] = llvm.sub %[[VAL_5]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_23:.*]] = llvm.mul %[[VAL_22]], %[[VAL_4]] overflow : i64 +// CHECK: %[[VAL_24:.*]] = llvm.sub %[[VAL_2]], %[[VAL_6]] overflow : i64 +// CHECK: %[[VAL_25:.*]] = llvm.add %[[VAL_23]], %[[VAL_24]] overflow : i64 +// CHECK: %[[VAL_26:.*]] = llvm.mul %[[VAL_25]], %[[VAL_21]] overflow : i64 +// CHECK: %[[VAL_27:.*]] = llvm.add %[[VAL_26]], %[[VAL_20]] overflow : i64 +// CHECK: %[[VAL_28:.*]] = llvm.mul %[[VAL_21]], %[[VAL_1]] overflow : i64 // CHECK: %[[VAL_30:.*]] = llvm.getelementptr %[[VAL_0]][%[[VAL_27]]] : (!llvm.ptr, i64) -> !llvm.ptr, i32 // CHECK: llvm.return // CHECK: } @@ -2193,13 +2193,13 @@ func.func @ext_array_coor_dt_slice(%arg0: !fir.ref : i64 +// CHECK: %[[VAL_10:.*]] = llvm.mul %[[VAL_9]], %[[VAL_4]] overflow : i64 +// CHECK: %[[VAL_11:.*]] = llvm.sub %[[VAL_2]], %[[VAL_7]] overflow : i64 +// CHECK: %[[VAL_12:.*]] = llvm.add %[[VAL_10]], %[[VAL_11]] overflow : i64 +// CHECK: %[[VAL_13:.*]] = llvm.mul %[[VAL_12]], %[[VAL_7]] overflow : i64 +// CHECK: %[[VAL_14:.*]] = llvm.add %[[VAL_13]], %[[VAL_8]] overflow : i64 +// CHECK: %[[VAL_15:.*]] = llvm.mul %[[VAL_7]], %[[VAL_1]] overflow : i64 // CHECK: %[[VAL_17:.*]] = llvm.getelementptr %[[VAL_0]][%[[VAL_14]], 0] : (!llvm.ptr, i64) -> !llvm.ptr, !llvm.struct<"_QFtest_dt_sliceTt", (i32, i32)> // CHECK: llvm.return // CHECK: } @@ -2452,8 +2452,8 @@ func.func @coordinate_box_array_1d(%arg0: !fir.box>, %arg1: // Index of the 1st CFI_dim_t object (corresonds the the 1st dimension) // CHECK-NEXT: %[[DIM_1_MEM_STRIDE_ADDR:.*]] = llvm.getelementptr %[[BOX]][0, 7, 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)> // CHECK-NEXT: %[[DIM_1_MEM_STRIDE_VAL:.*]] = llvm.load %[[DIM_1_MEM_STRIDE_ADDR]] : !llvm.ptr -> i64 -// CHECK-NEXT: %[[BYTE_OFFSET:.*]] = llvm.mul %[[COORDINATE]], %[[DIM_1_MEM_STRIDE_VAL]] : i64 -// CHECK-NEXT: %[[SUBOJECT_OFFSET:.*]] = llvm.add %[[BYTE_OFFSET]], %[[OFFSET_INIT]] : i64 +// CHECK-NEXT: %[[BYTE_OFFSET:.*]] = llvm.mul %[[COORDINATE]], %[[DIM_1_MEM_STRIDE_VAL]] overflow : i64 +// CHECK-NEXT: %[[SUBOJECT_OFFSET:.*]] = llvm.add %[[BYTE_OFFSET]], %[[OFFSET_INIT]] overflow : i64 // CHECK-NEXT: %[[SUBOBJECT_ADDR:.*]] = llvm.getelementptr %[[ARRAY_OBJECT]][%[[SUBOJECT_OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 // CHECK-NEXT: llvm.return @@ -2471,8 +2471,8 @@ func.func @coordinate_of_box_dynamic_array_1d(%arg0: !fir.box !llvm.ptr, !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<1 x array<3 x i64>>)> // CHECK-NEXT: %[[DIM_1_MEM_STRIDE_VAL:.*]] = llvm.load %[[DIM_1_MEM_STRIDE_ADDR]] : !llvm.ptr -> i64 -// CHECK-NEXT: %[[BYTE_OFFSET:.*]] = llvm.mul %[[COORDINATE]], %[[DIM_1_MEM_STRIDE_VAL]] : i64 -// CHECK-NEXT: %[[SUBOJECT_OFFSET:.*]] = llvm.add %[[BYTE_OFFSET]], %[[OFFSET_INIT]] : i64 +// CHECK-NEXT: %[[BYTE_OFFSET:.*]] = llvm.mul %[[COORDINATE]], %[[DIM_1_MEM_STRIDE_VAL]] overflow : i64 +// CHECK-NEXT: %[[SUBOJECT_OFFSET:.*]] = llvm.add %[[BYTE_OFFSET]], %[[OFFSET_INIT]] overflow : i64 // CHECK-NEXT: %[[SUBOBJECT_ADDR:.*]] = llvm.getelementptr %[[ARRAY_OBJECT]][%[[SUBOJECT_OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 // CHECK-NEXT: llvm.return @@ -2492,13 +2492,13 @@ func.func @coordinate_box_array_2d(%arg0: !fir.box>, % // Index of the 1st CFI_dim_t object (corresonds the the 1st dimension) // CHECK-NEXT: %[[DIM_1_MEM_STRIDE_ADDR:.*]] = llvm.getelementptr %[[BOX]][0, 7, 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<2 x array<3 x i64>>)> // CHECK-NEXT: %[[DIM_1_MEM_STRIDE_VAL:.*]] = llvm.load %[[DIM_1_MEM_STRIDE_ADDR]] : !llvm.ptr -> i64 -// CHECK-NEXT: %[[BYTE_OFFSET_1:.*]] = llvm.mul %[[COORDINATE_1]], %[[DIM_1_MEM_STRIDE_VAL]] : i64 -// CHECK-NEXT: %[[SUBOBJECT_OFFSET_1:.*]] = llvm.add %[[BYTE_OFFSET]], %[[OFFSET_INIT]] : i64 +// CHECK-NEXT: %[[BYTE_OFFSET_1:.*]] = llvm.mul %[[COORDINATE_1]], %[[DIM_1_MEM_STRIDE_VAL]] overflow : i64 +// CHECK-NEXT: %[[SUBOBJECT_OFFSET_1:.*]] = llvm.add %[[BYTE_OFFSET]], %[[OFFSET_INIT]] overflow : i64 // Index of the 1st CFI_dim_t object (corresonds the the 2nd dimension) // CHECK-NEXT: %[[DIM_2_MEM_STRIDE_ADDR:.*]] = llvm.getelementptr %[[BOX]][0, 7, 1, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, i{{.*}}, array<2 x array<3 x i64>>)> // CHECK-NEXT: %[[DIM_2_MEM_STRIDE_VAL:.*]] = llvm.load %[[DIM_2_MEM_STRIDE_ADDR]] : !llvm.ptr -> i64 -// CHECK-NEXT: %[[BYTE_OFFSET_2:.*]] = llvm.mul %[[COORDINATE_2]], %[[DIM_2_MEM_STRIDE_VAL]] : i64 -// CHECK-NEXT: %[[SUBOBJECT_OFFSET_2:.*]] = llvm.add %[[BYTE_OFFSET_2]], %[[SUBOBJECT_OFFSET_1]] : i64 +// CHECK-NEXT: %[[BYTE_OFFSET_2:.*]] = llvm.mul %[[COORDINATE_2]], %[[DIM_2_MEM_STRIDE_VAL]] overflow : i64 +// CHECK-NEXT: %[[SUBOBJECT_OFFSET_2:.*]] = llvm.add %[[BYTE_OFFSET_2]], %[[SUBOBJECT_OFFSET_1]] overflow : i64 // CHECK-NEXT: %[[SUBOBJECT_ADDR:.*]] = llvm.getelementptr %[[ARRAY_OBJECT]][%[[SUBOBJECT_OFFSET_2]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 // CHECK-NEXT: llvm.return @@ -2520,8 +2520,8 @@ func.func @coordinate_box_derived_inside_array(%arg0: !fir.box !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> // CHECK: %[[VAL_14:.*]] = llvm.load %[[VAL_13]] : !llvm.ptr -> i64 -// CHECK: %[[VAL_15:.*]] = llvm.mul %[[COORDINATE_1]], %[[VAL_14]] : i64 -// CHECK: %[[OFFSET:.*]] = llvm.add %[[VAL_15]], %[[VAL_8]] : i64 +// CHECK: %[[VAL_15:.*]] = llvm.mul %[[COORDINATE_1]], %[[VAL_14]] overflow : i64 +// CHECK: %[[OFFSET:.*]] = llvm.add %[[VAL_15]], %[[VAL_8]] overflow : i64 // CHECK: %[[DERIVED:.*]] = llvm.getelementptr %[[ARRAY]][%[[OFFSET]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 // CHECK: %[[VAL_20:.*]] = llvm.getelementptr %[[DERIVED]][0, 1] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<"derived_3", (f32, f32)> // CHECK: llvm.return diff --git a/flang/test/Fir/coordinateof.fir b/flang/test/Fir/coordinateof.fir index c605f43a1543..acb9fd57c095 100644 --- a/flang/test/Fir/coordinateof.fir +++ b/flang/test/Fir/coordinateof.fir @@ -30,8 +30,8 @@ func.func @foo3(%box : !fir.box>, %i : i32) -> i32 { // CHECK: %[[boxptr:.*]] = load ptr, ptr %[[gep0]] // CHECK: %[[gep1:.*]] = getelementptr { ptr, i64, {{.*}} i32 7 // CHECK: %[[stride:.*]] = load i64, ptr %[[gep1]] - // CHECK: %[[dimoffset:.*]] = mul i64 %[[cvt]], %[[stride]] - // CHECK: %[[offset:.*]] = add i64 %[[dimoffset]], 0 + // CHECK: %[[dimoffset:.*]] = mul nsw i64 %[[cvt]], %[[stride]] + // CHECK: %[[offset:.*]] = add nsw i64 %[[dimoffset]], 0 // CHECK: %[[gep2:.*]] = getelementptr i8, ptr %[[boxptr]], i64 %[[offset]] %1 = fir.coordinate_of %box, %ii : (!fir.box>, index) -> !fir.ref // CHECK: load i32, ptr %[[gep2]] @@ -68,8 +68,8 @@ func.func @foo6(%box : !fir.box>>>, %i : i64 // CHECK: %[[addr:.*]] = load ptr, ptr %[[addr_gep]] // CHECK: %[[stride_gep:.*]] = getelementptr { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[box]], i32 0, i32 7, i32 0, i32 2 // CHECK: %[[stride:.*]] = load i64, ptr %[[stride_gep]] - // CHECK: %[[mul:.*]] = mul i64 %{{.*}}, %[[stride]] - // CHECK: %[[offset:.*]] = add i64 %[[mul]], 0 + // CHECK: %[[mul:.*]] = mul nsw i64 %{{.*}}, %[[stride]] + // CHECK: %[[offset:.*]] = add nsw i64 %[[mul]], 0 // CHECK: %[[gep:.*]] = getelementptr i8, ptr %[[addr]], i64 %[[offset]] %coor = fir.coordinate_of %box, %i : (!fir.box>>>, i64) -> !fir.ref> diff --git a/flang/test/Fir/tbaa.fir b/flang/test/Fir/tbaa.fir index bbc8a81bdf68..4474bbbe3dd7 100644 --- a/flang/test/Fir/tbaa.fir +++ b/flang/test/Fir/tbaa.fir @@ -41,8 +41,8 @@ module { // CHECK: %[[VAL_8:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_9:.*]] = llvm.getelementptr %[[VAL_0]][0, 7, 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>, ptr, array<1 x i64>)> // CHECK: %[[VAL_10:.*]] = llvm.load %[[VAL_9]] {tbaa = [#[[$BOXT]]]} : !llvm.ptr -> i64 -// CHECK: %[[VAL_11:.*]] = llvm.mul %[[VAL_4]], %[[VAL_10]] : i64 -// CHECK: %[[VAL_12:.*]] = llvm.add %[[VAL_11]], %[[VAL_8]] : i64 +// CHECK: %[[VAL_11:.*]] = llvm.mul %[[VAL_4]], %[[VAL_10]] overflow : i64 +// CHECK: %[[VAL_12:.*]] = llvm.add %[[VAL_11]], %[[VAL_8]] overflow : i64 // CHECK: %[[VAL_14:.*]] = llvm.getelementptr %[[VAL_7]]{{\[}}%[[VAL_12]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 // CHECK: %[[VAL_16:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_17:.*]] = llvm.mlir.constant(-1 : i32) : i32 @@ -330,12 +330,12 @@ func.func @tbaa(%arg0: !fir.box>) { // CHECK: %[[VAL_1:.*]] = llvm.mlir.constant(0 : i64) : i64 // CHECK: %[[VAL_2:.*]] = llvm.mlir.constant(1 : i64) : i64 // CHECK: %[[VAL_3:.*]] = llvm.mlir.constant(0 : i64) : i64 -// CHECK: %[[VAL_4:.*]] = llvm.sub %[[VAL_1]], %[[VAL_2]] : i64 -// CHECK: %[[VAL_5:.*]] = llvm.mul %[[VAL_4]], %[[VAL_2]] : i64 +// CHECK: %[[VAL_4:.*]] = llvm.sub %[[VAL_1]], %[[VAL_2]] overflow : i64 +// CHECK: %[[VAL_5:.*]] = llvm.mul %[[VAL_4]], %[[VAL_2]] overflow : i64 // CHECK: %[[VAL_6:.*]] = llvm.getelementptr %[[VAL_0]][0, 7, 0, 2] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> // CHECK: %[[VAL_7:.*]] = llvm.load %[[VAL_6]] {tbaa = [#[[$BOXT]]]} : !llvm.ptr -> i64 -// CHECK: %[[VAL_8:.*]] = llvm.mul %[[VAL_5]], %[[VAL_7]] : i64 -// CHECK: %[[VAL_9:.*]] = llvm.add %[[VAL_8]], %[[VAL_3]] : i64 +// CHECK: %[[VAL_8:.*]] = llvm.mul %[[VAL_5]], %[[VAL_7]] overflow : i64 +// CHECK: %[[VAL_9:.*]] = llvm.add %[[VAL_8]], %[[VAL_3]] overflow : i64 // CHECK: %[[VAL_10:.*]] = llvm.getelementptr %[[VAL_0]][0, 0] : (!llvm.ptr) -> !llvm.ptr, !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> // CHECK: %[[VAL_11:.*]] = llvm.load %[[VAL_10]] {tbaa = [#[[$BOXT]]]} : !llvm.ptr -> !llvm.ptr // CHECK: %[[VAL_13:.*]] = llvm.getelementptr %[[VAL_11]]{{\[}}%[[VAL_9]]] : (!llvm.ptr, i64) -> !llvm.ptr, i8 diff --git a/flang/test/Lower/HLFIR/implicit-type-conversion-allocatable.f90 b/flang/test/Lower/HLFIR/implicit-type-conversion-allocatable.f90 index 7083a825dfd3..361cd61adea2 100644 --- a/flang/test/Lower/HLFIR/implicit-type-conversion-allocatable.f90 +++ b/flang/test/Lower/HLFIR/implicit-type-conversion-allocatable.f90 @@ -38,3 +38,29 @@ end subroutine ! CHECK: hlfir.destroy %[[VAL_8]] : !hlfir.expr ! CHECK: return ! CHECK: } + +! Test that RHS character length is preserved in a character KIND +! conversion before the assignment. +subroutine kind_and_length(a, b) + character(len=4,kind=4), allocatable :: a(:) + character(len=2,kind=1) :: b(:) + a = b +end subroutine +! CHECK-LABEL: func.func @_QPkind_and_length( +! CHECK: %[[VAL_2:.*]] = arith.constant 4 : index +! CHECK: %[[VAL_3:.*]]:2 = hlfir.declare %[[VAL_0:[a-z0-9]*]] typeparams %[[VAL_2:[a-z0-9]*]] {{.*}}Ea +! CHECK: %[[VAL_4:.*]] = arith.constant 2 : index +! CHECK: %[[VAL_5:.*]]:2 = hlfir.declare %[[VAL_1:[a-z0-9]*]] typeparams %[[VAL_4:[a-z0-9]*]] {{.*}}Eb +! CHECK: %[[VAL_6:.*]] = arith.constant 0 : index +! CHECK: %[[VAL_7:.*]]:3 = fir.box_dims %[[VAL_5]]#0, %[[VAL_6]] : (!fir.box>>, index) -> (index, index, index) +! CHECK: %[[VAL_8:.*]] = fir.shape %[[VAL_7]]#1 : (index) -> !fir.shape<1> +! CHECK: %[[VAL_9:.*]] = hlfir.elemental %[[VAL_8]] typeparams %[[VAL_4]] unordered : (!fir.shape<1>, index) -> !hlfir.expr> { +! CHECK: ^bb0(%[[VAL_10:.*]]: index): +! CHECK: %[[VAL_11:.*]] = hlfir.designate %[[VAL_5]]#0 (%[[VAL_10]]) typeparams %[[VAL_4]] : (!fir.box>>, index, index) -> !fir.ref> +! CHECK: %[[VAL_12:.*]] = fir.alloca !fir.char<4,?>(%[[VAL_4]] : index) +! CHECK: fir.char_convert %[[VAL_11]] for %[[VAL_4]] to %[[VAL_12]] : !fir.ref>, index, !fir.ref> +! CHECK: %[[VAL_13:.*]]:2 = hlfir.declare %[[VAL_12:[a-z0-9]*]] typeparams %[[VAL_4:[a-z0-9]*]] {uniq_name = ".temp.kindconvert"} : (!fir.ref>, index) -> (!fir.boxchar<4>, !fir.ref>) +! CHECK: hlfir.yield_element %[[VAL_13]]#0 : !fir.boxchar<4> +! CHECK: } +! CHECK: hlfir.assign %[[VAL_9]] to %[[VAL_3]]#0 realloc keep_lhs_len : !hlfir.expr>, !fir.ref>>>> + diff --git a/flang/test/Lower/OpenACC/acc-set.f90 b/flang/test/Lower/OpenACC/acc-set.f90 index 2741c3c63255..31c1912a87f5 100644 --- a/flang/test/Lower/OpenACC/acc-set.f90 +++ b/flang/test/Lower/OpenACC/acc-set.f90 @@ -33,7 +33,7 @@ end ! CHECK: %[[C0:.*]] = arith.constant 0 : i32 ! CHECK: acc.set device_num(%[[C0]] : i32) -! CHECK: acc.set attributes {device_type = #acc.device_type<*>} +! CHECK: acc.set attributes {device_type = #acc.device_type} ! CHECK: acc.set attributes {device_type = #acc.device_type} diff --git a/flang/test/Lower/OpenACC/acc-update.f90 b/flang/test/Lower/OpenACC/acc-update.f90 index 2e8d672a43eb..d2b15f8bd258 100644 --- a/flang/test/Lower/OpenACC/acc-update.f90 +++ b/flang/test/Lower/OpenACC/acc-update.f90 @@ -114,7 +114,7 @@ subroutine acc_update !$acc update host(a) device_type(*) ! CHECK: %[[DEVPTR_A:.*]] = acc.getdeviceptr varPtr(%[[DECLA]]#1 : !fir.ref>) bounds(%{{.*}}, %{{.*}}) -> !fir.ref> {dataClause = #acc, name = "a", structured = false} -! CHECK: acc.update dataOperands(%[[DEVPTR_A]] : !fir.ref>) attributes {device_types = [#acc.device_type<*>]} +! CHECK: acc.update dataOperands(%[[DEVPTR_A]] : !fir.ref>) attributes {device_types = [#acc.device_type]} ! CHECK: acc.update_host accPtr(%[[DEVPTR_A]] : !fir.ref>) bounds(%{{.*}}, %{{.*}}) to varPtr(%[[DECLA]]#1 : !fir.ref>) {name = "a", structured = false} end subroutine acc_update diff --git a/flang/test/Lower/PowerPC/ppc-vec-store-elem-order.f90 b/flang/test/Lower/PowerPC/ppc-vec-store-elem-order.f90 index 494ed21f4fe9..caf6d5463a83 100644 --- a/flang/test/Lower/PowerPC/ppc-vec-store-elem-order.f90 +++ b/flang/test/Lower/PowerPC/ppc-vec-store-elem-order.f90 @@ -67,10 +67,10 @@ subroutine vec_xstd2_test(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x float>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x float>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i16, ptr %1, align 2 @@ -93,10 +93,10 @@ subroutine vec_xstw4_test(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x float>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x float>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i16, ptr %1, align 2 diff --git a/flang/test/Lower/PowerPC/ppc-vec-store.f90 b/flang/test/Lower/PowerPC/ppc-vec-store.f90 index 8e20228d6825..c25cc8b07cf7 100644 --- a/flang/test/Lower/PowerPC/ppc-vec-store.f90 +++ b/flang/test/Lower/PowerPC/ppc-vec-store.f90 @@ -89,10 +89,10 @@ subroutine vec_st_vi4i4via4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[iextsub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[iextmul:.*]] = mul i64 %[[iextsub]], 1 -! LLVMIR: %[[iextmul2:.*]] = mul i64 %[[iextmul]], 1 -! LLVMIR: %[[iextadd:.*]] = add i64 %[[iextmul2]], 0 +! LLVMIR: %[[iextsub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[iextmul:.*]] = mul nsw i64 %[[iextsub]], 1 +! LLVMIR: %[[iextmul2:.*]] = mul nsw i64 %[[iextmul]], 1 +! LLVMIR: %[[iextadd:.*]] = add nsw i64 %[[iextmul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x i32>, ptr %2, i64 %[[iextadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 @@ -206,10 +206,10 @@ subroutine vec_ste_vi4i4ia4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr i32, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 @@ -244,10 +244,10 @@ subroutine vec_stxv_test_vi4i8ia4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr i32, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i64, ptr %1, align 8 @@ -278,10 +278,10 @@ subroutine vec_stxv_test_vi4i4vai4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x i32>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 @@ -317,10 +317,10 @@ subroutine vec_xst_test_vi4i8ia4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr i32, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i64, ptr %1, align 8 @@ -351,10 +351,10 @@ subroutine vec_xst_test_vi4i4vai4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x i32>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 @@ -390,10 +390,10 @@ subroutine vec_xst_be_test_vi4i8ia4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr i32, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i64, ptr %1, align 8 @@ -426,10 +426,10 @@ subroutine vec_xst_be_test_vi4i4vai4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x i32>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 @@ -467,10 +467,10 @@ subroutine vec_xstd2_test_vi4i8ia4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr i32, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i64, ptr %1, align 8 @@ -503,10 +503,10 @@ subroutine vec_xstd2_test_vi4i4vai4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x i32>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 @@ -543,10 +543,10 @@ subroutine vec_xstw4_test_vi4i8ia4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr i32, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i64, ptr %1, align 8 @@ -578,10 +578,10 @@ subroutine vec_xstw4_test_vi4i4vai4(arg1, arg2, arg3, i) ! LLVMIR: %[[i:.*]] = load i32, ptr %3, align 4 ! LLVMIR: %[[iext:.*]] = sext i32 %[[i]] to i64 -! LLVMIR: %[[isub:.*]] = sub i64 %[[iext]], 1 -! LLVMIR: %[[imul1:.*]] = mul i64 %[[isub]], 1 -! LLVMIR: %[[imul2:.*]] = mul i64 %[[imul1]], 1 -! LLVMIR: %[[iadd:.*]] = add i64 %[[imul2]], 0 +! LLVMIR: %[[isub:.*]] = sub nsw i64 %[[iext]], 1 +! LLVMIR: %[[imul1:.*]] = mul nsw i64 %[[isub]], 1 +! LLVMIR: %[[imul2:.*]] = mul nsw i64 %[[imul1]], 1 +! LLVMIR: %[[iadd:.*]] = add nsw i64 %[[imul2]], 0 ! LLVMIR: %[[gep1:.*]] = getelementptr <4 x i32>, ptr %2, i64 %[[iadd]] ! LLVMIR: %[[arg1:.*]] = load <4 x i32>, ptr %0, align 16 ! LLVMIR: %[[arg2:.*]] = load i32, ptr %1, align 4 diff --git a/flang/test/Lower/RISCV/riscv-vector-bits-vscale-range.f90 b/flang/test/Lower/RISCV/riscv-vector-bits-vscale-range.f90 new file mode 100644 index 000000000000..b08ad91feb38 --- /dev/null +++ b/flang/test/Lower/RISCV/riscv-vector-bits-vscale-range.f90 @@ -0,0 +1,21 @@ +! REQUIRES: riscv-registered-target +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=1 -mvscale-max=1 -emit-llvm -o - %s | FileCheck %s -D#VBITS=1 +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=2 -mvscale-max=2 -emit-llvm -o - %s | FileCheck %s -D#VBITS=2 +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=4 -mvscale-max=4 -emit-llvm -o - %s | FileCheck %s -D#VBITS=4 +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=8 -mvscale-max=8 -emit-llvm -o - %s | FileCheck %s -D#VBITS=8 +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=16 -mvscale-max=16 -emit-llvm -o - %s | FileCheck %s -D#VBITS=16 +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=1 -emit-llvm -o - %s | FileCheck %s -D#VBITS=1 --check-prefix=CHECK-NOMAX +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=2 -emit-llvm -o - %s | FileCheck %s -D#VBITS=2 --check-prefix=CHECK-NOMAX +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=4 -emit-llvm -o - %s | FileCheck %s -D#VBITS=4 --check-prefix=CHECK-NOMAX +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=8 -emit-llvm -o - %s | FileCheck %s -D#VBITS=8 --check-prefix=CHECK-NOMAX +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=16 -emit-llvm -o - %s | FileCheck %s -D#VBITS=16 --check-prefix=CHECK-NOMAX +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -mvscale-min=1 -mvscale-max=0 -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-UNBOUNDED +! RUN: %flang_fc1 -triple riscv64-none-linux-gnu -target-feature +v -emit-llvm -o - %s | FileCheck %s --check-prefix=CHECK-NONE + +! CHECK-LABEL: @func_() #0 +! CHECK: attributes #0 = {{{.*}} vscale_range([[#VBITS]],[[#VBITS]]) {{.*}}} +! CHECK-NOMAX: attributes #0 = {{{.*}} vscale_range([[#VBITS]],0) {{.*}}} +! CHECK-UNBOUNDED: attributes #0 = {{{.*}} vscale_range(1,0) {{.*}}} +! CHECK-NONE-NOT: vscale_range +subroutine func +end subroutine func diff --git a/flang/test/Parser/continuation-before-quote.f90 b/flang/test/Parser/continuation-before-quote.f90 deleted file mode 100644 index 66252010d89c..000000000000 --- a/flang/test/Parser/continuation-before-quote.f90 +++ /dev/null @@ -1,10 +0,0 @@ -! RUN: %flang_fc1 -fsyntax-only -pedantic %s 2>&1 | FileCheck %s -! Continuation between repeated quotation marks -subroutine test -!CHECK: portability: Repeated quote mark in character literal continuation line should have been preceded by '&' - print *, 'needs an '& -'ampersand''' -!CHECK-NOT: portability: Repeated quote mark in character literal continuation line should have been preceded by '&' - print *, 'has an '& -&'ampersand''' -end diff --git a/flang/test/Parser/continuation-without-ampersand.f90 b/flang/test/Parser/continuation-without-ampersand.f90 new file mode 100644 index 000000000000..5c3f23235edc --- /dev/null +++ b/flang/test/Parser/continuation-without-ampersand.f90 @@ -0,0 +1,13 @@ +! RUN: %flang_fc1 -fsyntax-only -pedantic %s 2>&1 | FileCheck %s +! Continuation between repeated quotation marks +subroutine test +!CHECK: portability: Character literal continuation line should have been preceded by '&' + print *, 'needs an '& +'ampersand''' +!CHECK: portability: Character literal continuation line should have been preceded by '&' + print *, 'also needs an '& + 'ampersand''' +!CHECK-NOT: portability: Character literal continuation line should have been preceded by '&' + print *, 'has an '& +&'ampersand''' +end diff --git a/flang/test/Semantics/OpenMP/target01.f90 b/flang/test/Semantics/OpenMP/target01.f90 index 485fa1f2530c..9836f0112738 100644 --- a/flang/test/Semantics/OpenMP/target01.f90 +++ b/flang/test/Semantics/OpenMP/target01.f90 @@ -39,19 +39,19 @@ subroutine bar(b1, b2, b3) type(c_ptr), pointer :: b2 type(c_ptr), value :: b3 - !ERROR: Variable 'c' in IS_DEVICE_PTR clause must be a dummy argument + !WARNING: Variable 'c' in IS_DEVICE_PTR clause must be a dummy argument. This semantic check is deprecated from OpenMP 5.2 and later. !$omp target is_device_ptr(c) y = y + 1 !$omp end target - !ERROR: Variable 'b1' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. + !WARNING: Variable 'b1' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later. !$omp target is_device_ptr(b1) y = y + 1 !$omp end target - !ERROR: Variable 'b2' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. + !WARNING: Variable 'b2' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later. !$omp target is_device_ptr(b2) y = y + 1 !$omp end target - !ERROR: Variable 'b3' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. + !WARNING: Variable 'b3' in IS_DEVICE_PTR clause must be a dummy argument that does not have the ALLOCATABLE, POINTER or VALUE attribute. This semantic check is deprecated from OpenMP 5.2 and later. !$omp target is_device_ptr(b3) y = y + 1 !$omp end target diff --git a/flang/test/Semantics/definable01.f90 b/flang/test/Semantics/definable01.f90 index fff493fe7a41..c0f10668fb48 100644 --- a/flang/test/Semantics/definable01.f90 +++ b/flang/test/Semantics/definable01.f90 @@ -82,4 +82,19 @@ module m subroutine test3b(pp) procedure(sin), pointer, intent(in out) :: pp end subroutine + subroutine test4(p) + type(ptype), pointer, intent(in) :: p + p%x = 1. + p%ptr = 1. ! ok + nullify(p%ptr) ! ok + !CHECK: error: 'p' may not appear in NULLIFY + !CHECK: because: 'p' is an INTENT(IN) dummy argument + nullify(p) + end + subroutine test5(np) + type(ptype), intent(in) :: np + !CHECK: error: 'ptr' may not appear in NULLIFY + !CHECK: because: 'np' is an INTENT(IN) dummy argument + nullify(np%ptr) + end end module diff --git a/flang/unittests/Runtime/NumericalFormatTest.cpp b/flang/unittests/Runtime/NumericalFormatTest.cpp index 219947fe4fbb..b5b8eb059437 100644 --- a/flang/unittests/Runtime/NumericalFormatTest.cpp +++ b/flang/unittests/Runtime/NumericalFormatTest.cpp @@ -710,8 +710,12 @@ TEST(IOApiTests, FormatDoubleValues) { {"(F5.3,';')", 0.099999, "0.100;"}, {"(F5.3,';')", 0.0099999, "0.010;"}, {"(F5.3,';')", 0.00099999, "0.001;"}, - {"(F5.3,';')", 0.0005, "0.001;"}, - {"(F5.3,';')", 0.00049999, "0.000;"}, + {"(F5.3,';')", + 0.0005000000000000000104083408558608425664715468883514404296875, + "0.001;"}, + {"(F5.3,';')", + 0.000499999999999999901988123607310399165726266801357269287109375, + "0.000;"}, {"(F5.3,';')", 0.000099999, "0.000;"}, {"(F5.3,';')", -99.999, "*****;"}, {"(F5.3,';')", -9.9999, "*****;"}, @@ -719,17 +723,30 @@ TEST(IOApiTests, FormatDoubleValues) { {"(F5.3,';')", -0.099999, "-.100;"}, {"(F5.3,';')", -0.0099999, "-.010;"}, {"(F5.3,';')", -0.00099999, "-.001;"}, - {"(F5.3,';')", -0.0005, "-.001;"}, - {"(F5.3,';')", -0.00049999, "-.000;"}, + {"(F5.3,';')", + -0.0005000000000000000104083408558608425664715468883514404296875, + "-.001;"}, + {"(F5.3,';')", + -0.000499999999999999901988123607310399165726266801357269287109375, + "-.000;"}, {"(F5.3,';')", -0.000099999, "-.000;"}, {"(F0.1,';')", 0.0, ".0;"}, + {"(F5.0,';')", -0.5000000000000001, " -1.;"}, + {"(F5.0,';')", -0.5, " -0.;"}, + {"(F5.0,';')", -0.49999999999999994, " -0.;"}, + {"(F5.0,';')", 0.49999999999999994, " 0.;"}, + {"(F5.0,';')", 0.5, " 0.;"}, + {"(F5.0,';')", 0.5000000000000001, " 1.;"}, }; for (auto const &[format, value, expect] : individualTestCases) { std::string got; + char hex[17]; + std::snprintf(hex, sizeof hex, "%016llx", + *reinterpret_cast(&value)); ASSERT_TRUE(CompareFormatReal(format, value, expect, got)) - << "Failed to format " << format << ", expected '" << expect - << "', got '" << got << "'"; + << "Failed to format " << value << " 0x" << hex << " with format " + << format << ", expected '" << expect << "', got '" << got << "'"; } // Problematic EN formatting edge cases with rounding diff --git a/libc/src/__support/FPUtil/FPBits.h b/libc/src/__support/FPUtil/FPBits.h index ca98aa712624..bd075fe3d728 100644 --- a/libc/src/__support/FPUtil/FPBits.h +++ b/libc/src/__support/FPUtil/FPBits.h @@ -59,13 +59,13 @@ template struct FPBits { return bits & FloatProp::MANTISSA_MASK; } - LIBC_INLINE constexpr void set_unbiased_exponent(UIntType expVal) { + LIBC_INLINE constexpr void set_biased_exponent(UIntType expVal) { expVal = (expVal << (FloatProp::MANTISSA_WIDTH)) & FloatProp::EXPONENT_MASK; bits &= ~(FloatProp::EXPONENT_MASK); bits |= expVal; } - LIBC_INLINE constexpr uint16_t get_unbiased_exponent() const { + LIBC_INLINE constexpr uint16_t get_biased_exponent() const { return uint16_t((bits & FloatProp::EXPONENT_MASK) >> (FloatProp::MANTISSA_WIDTH)); } @@ -73,7 +73,7 @@ template struct FPBits { // The function return mantissa with the implicit bit set iff the current // value is a valid normal number. LIBC_INLINE constexpr UIntType get_explicit_mantissa() { - return ((get_unbiased_exponent() > 0 && !is_inf_or_nan()) + return ((get_biased_exponent() > 0 && !is_inf_or_nan()) ? (FloatProp::MANTISSA_MASK + 1) : 0) | (FloatProp::MANTISSA_MASK & bits); @@ -126,7 +126,7 @@ template struct FPBits { LIBC_INLINE constexpr UIntType uintval() const { return bits; } LIBC_INLINE constexpr int get_exponent() const { - return int(get_unbiased_exponent()) - EXPONENT_BIAS; + return int(get_biased_exponent()) - EXPONENT_BIAS; } // If the number is subnormal, the exponent is treated as if it were the @@ -136,13 +136,13 @@ template struct FPBits { // will give a slightly incorrect result. Additionally, zero has an exponent // of zero, and that should actually be treated as zero. LIBC_INLINE constexpr int get_explicit_exponent() const { - const int unbiased_exp = int(get_unbiased_exponent()); + const int biased_exp = int(get_biased_exponent()); if (is_zero()) { return 0; - } else if (unbiased_exp == 0) { + } else if (biased_exp == 0) { return 1 - EXPONENT_BIAS; } else { - return unbiased_exp - EXPONENT_BIAS; + return biased_exp - EXPONENT_BIAS; } } @@ -228,7 +228,7 @@ template struct FPBits { if (LIBC_LIKELY(ep >= 0)) { // Implicit number bit will be removed by mask result.set_mantissa(number); - result.set_unbiased_exponent(ep + 1); + result.set_biased_exponent(ep + 1); } else { result.set_mantissa(number >> -ep); } @@ -236,10 +236,10 @@ template struct FPBits { } LIBC_INLINE static constexpr FPBits - create_value(bool sign, UIntType unbiased_exp, UIntType mantissa) { + create_value(bool sign, UIntType biased_exp, UIntType mantissa) { FPBits result; result.set_sign(sign); - result.set_unbiased_exponent(unbiased_exp); + result.set_biased_exponent(biased_exp); result.set_mantissa(mantissa); return result; } diff --git a/libc/src/__support/FPUtil/FloatProperties.h b/libc/src/__support/FPUtil/FloatProperties.h index 59e261e1047f..ef6a3cd403db 100644 --- a/libc/src/__support/FPUtil/FloatProperties.h +++ b/libc/src/__support/FPUtil/FloatProperties.h @@ -10,7 +10,7 @@ #define LLVM_LIBC_SRC___SUPPORT_FPUTIL_FLOATPROPERTIES_H #include "src/__support/UInt128.h" -#include "src/__support/macros/attributes.h" // LIBC_INLINE +#include "src/__support/macros/attributes.h" // LIBC_INLINE, LIBC_INLINE_VAR #include "src/__support/macros/properties/float.h" // LIBC_COMPILER_HAS_FLOAT128 #include @@ -27,116 +27,177 @@ enum class FPType { X86_Binary80, }; -template struct FPProperties {}; -template <> struct FPProperties { - typedef uint32_t BitsType; +// For now 'FPEncoding', 'FPBaseProperties' and 'FPCommonProperties' are +// implementation details. +namespace internal { - static constexpr uint32_t BIT_WIDTH = sizeof(BitsType) * 8; - - static constexpr uint32_t MANTISSA_WIDTH = 23; - // The mantissa precision includes the implicit bit. - static constexpr uint32_t MANTISSA_PRECISION = MANTISSA_WIDTH + 1; - static constexpr uint32_t EXPONENT_WIDTH = 8; - static constexpr BitsType MANTISSA_MASK = (BitsType(1) << MANTISSA_WIDTH) - 1; - static constexpr BitsType SIGN_MASK = BitsType(1) - << (EXPONENT_WIDTH + MANTISSA_WIDTH); - static constexpr BitsType EXPONENT_MASK = ~(SIGN_MASK | MANTISSA_MASK); - static constexpr uint32_t EXPONENT_BIAS = 127; - - static constexpr BitsType EXP_MANT_MASK = MANTISSA_MASK + EXPONENT_MASK; - static_assert(EXP_MANT_MASK == ~SIGN_MASK, - "Exponent and mantissa masks are not as expected."); - - // If a number x is a NAN, then it is a quiet NAN if: - // QuietNaNMask & bits(x) != 0 - // Else, it is a signalling NAN. - static constexpr BitsType QUIET_NAN_MASK = 0x00400000U; +// The type of encoding for supported floating point types. +enum class FPEncoding { + IEEE754, + X86_ExtendedPrecision, }; -template <> struct FPProperties { - typedef uint64_t BitsType; - - static constexpr uint32_t BIT_WIDTH = sizeof(BitsType) * 8; - - static constexpr uint32_t MANTISSA_WIDTH = 52; - static constexpr uint32_t MANTISSA_PRECISION = MANTISSA_WIDTH + 1; - static constexpr uint32_t EXPONENT_WIDTH = 11; - static constexpr BitsType MANTISSA_MASK = (BitsType(1) << MANTISSA_WIDTH) - 1; - static constexpr BitsType SIGN_MASK = BitsType(1) - << (EXPONENT_WIDTH + MANTISSA_WIDTH); - static constexpr BitsType EXPONENT_MASK = ~(SIGN_MASK | MANTISSA_MASK); - static constexpr uint32_t EXPONENT_BIAS = 1023; +template struct FPBaseProperties {}; - static constexpr BitsType EXP_MANT_MASK = MANTISSA_MASK + EXPONENT_MASK; - static_assert(EXP_MANT_MASK == ~SIGN_MASK, - "Exponent and mantissa masks are not as expected."); +template <> struct FPBaseProperties { + using UIntType = uint16_t; + LIBC_INLINE_VAR static constexpr int TOTAL_BITS = 16; + LIBC_INLINE_VAR static constexpr int SIG_BITS = 10; + LIBC_INLINE_VAR static constexpr int EXP_BITS = 5; + LIBC_INLINE_VAR static constexpr auto ENCODING = FPEncoding::IEEE754; +}; - // If a number x is a NAN, then it is a quiet NAN if: - // QuietNaNMask & bits(x) != 0 - // Else, it is a signalling NAN. - static constexpr BitsType QUIET_NAN_MASK = 0x0008000000000000ULL; +template <> struct FPBaseProperties { + using UIntType = uint32_t; + LIBC_INLINE_VAR static constexpr int TOTAL_BITS = 32; + LIBC_INLINE_VAR static constexpr int SIG_BITS = 23; + LIBC_INLINE_VAR static constexpr int EXP_BITS = 8; + LIBC_INLINE_VAR static constexpr auto ENCODING = FPEncoding::IEEE754; }; -// Properties for numbers represented in 80 bits long double on non-Windows x86 -// platforms. -template <> struct FPProperties { - typedef UInt128 BitsType; +template <> struct FPBaseProperties { + using UIntType = uint64_t; + LIBC_INLINE_VAR static constexpr int TOTAL_BITS = 64; + LIBC_INLINE_VAR static constexpr int SIG_BITS = 52; + LIBC_INLINE_VAR static constexpr int EXP_BITS = 11; + LIBC_INLINE_VAR static constexpr auto ENCODING = FPEncoding::IEEE754; +}; - static constexpr uint32_t BIT_WIDTH = (sizeof(BitsType) * 8) - 48; - static constexpr BitsType FULL_WIDTH_MASK = ((BitsType(1) << BIT_WIDTH) - 1); +template <> struct FPBaseProperties { + using UIntType = UInt128; + LIBC_INLINE_VAR static constexpr int TOTAL_BITS = 128; + LIBC_INLINE_VAR static constexpr int SIG_BITS = 112; + LIBC_INLINE_VAR static constexpr int EXP_BITS = 15; + LIBC_INLINE_VAR static constexpr auto ENCODING = FPEncoding::IEEE754; +}; - static constexpr uint32_t MANTISSA_WIDTH = 63; - static constexpr uint32_t MANTISSA_PRECISION = MANTISSA_WIDTH + 1; - static constexpr uint32_t EXPONENT_WIDTH = 15; - static constexpr BitsType MANTISSA_MASK = (BitsType(1) << MANTISSA_WIDTH) - 1; +template <> struct FPBaseProperties { + using UIntType = UInt128; + LIBC_INLINE_VAR static constexpr int TOTAL_BITS = 80; + LIBC_INLINE_VAR static constexpr int SIG_BITS = 64; + LIBC_INLINE_VAR static constexpr int EXP_BITS = 15; + LIBC_INLINE_VAR static constexpr auto ENCODING = + FPEncoding::X86_ExtendedPrecision; +}; - // The x86 80 bit float represents the leading digit of the mantissa - // explicitly. This is the mask for that bit. - static constexpr BitsType EXPLICIT_BIT_MASK = (BitsType(1) << MANTISSA_WIDTH); +// TODO: Move this utility elsewhere. +template static constexpr T mask_trailing_ones() { + static_assert(cpp::is_unsigned_v); + constexpr unsigned t_bits = CHAR_BIT * sizeof(T); + static_assert(count <= t_bits && "Invalid bit index"); + // It's important not to initialize T with -1, since T may be BigInt which + // will take -1 as a uint64_t and only initialize the low 64 bits. + return count == 0 ? 0 : ((~T(0)) >> (t_bits - count)); +} - static constexpr BitsType SIGN_MASK = - BitsType(1) << (EXPONENT_WIDTH + MANTISSA_WIDTH + 1); - static constexpr BitsType EXPONENT_MASK = - ((BitsType(1) << EXPONENT_WIDTH) - 1) << (MANTISSA_WIDTH + 1); - static constexpr uint32_t EXPONENT_BIAS = 16383; +// Derives more properties from 'FPBaseProperties' above. +// This class serves as a halfway point between 'FPBaseProperties' and +// 'FPProperties' below. +template +struct FPCommonProperties : private FPBaseProperties { +private: + using UP = FPBaseProperties; + using UP::EXP_BITS; + using UP::SIG_BITS; + using UP::TOTAL_BITS; + using UIntType = typename UP::UIntType; + + LIBC_INLINE_VAR static constexpr int STORAGE_BITS = + sizeof(UIntType) * CHAR_BIT; + static_assert(STORAGE_BITS >= TOTAL_BITS); + + // The number of bits to represent sign. + // For documentation purpose, always 1. + LIBC_INLINE_VAR static constexpr int SIGN_BITS = 1; + static_assert(SIGN_BITS + EXP_BITS + SIG_BITS == TOTAL_BITS); + + // The exponent bias. Always positive. + LIBC_INLINE_VAR static constexpr int32_t EXP_BIAS = + (1U << (EXP_BITS - 1U)) - 1U; + static_assert(EXP_BIAS > 0); + + // Shifts + LIBC_INLINE_VAR static constexpr int SIG_MASK_SHIFT = 0; + LIBC_INLINE_VAR static constexpr int EXP_MASK_SHIFT = SIG_BITS; + LIBC_INLINE_VAR static constexpr int SIGN_MASK_SHIFT = SIG_BITS + EXP_BITS; + + // Masks + LIBC_INLINE_VAR static constexpr UIntType SIG_MASK = + mask_trailing_ones() << SIG_MASK_SHIFT; + LIBC_INLINE_VAR static constexpr UIntType EXP_MASK = + mask_trailing_ones() << EXP_MASK_SHIFT; + // Trailing underscore on SIGN_MASK_ is temporary - it will be removed + // once we can replace the public part below with the private one. + LIBC_INLINE_VAR static constexpr UIntType SIGN_MASK_ = + mask_trailing_ones() << SIGN_MASK_SHIFT; + LIBC_INLINE_VAR static constexpr UIntType FP_MASK = + mask_trailing_ones(); + static_assert((SIG_MASK & EXP_MASK & SIGN_MASK_) == 0, "masks disjoint"); + static_assert((SIG_MASK | EXP_MASK | SIGN_MASK_) == FP_MASK, "masks cover"); + + LIBC_INLINE static constexpr UIntType bit_at(int position) { + return UIntType(1) << position; + } - static constexpr BitsType EXP_MANT_MASK = - MANTISSA_MASK | EXPLICIT_BIT_MASK | EXPONENT_MASK; - static_assert(EXP_MANT_MASK == (~SIGN_MASK & FULL_WIDTH_MASK), - "Exponent and mantissa masks are not as expected."); + LIBC_INLINE_VAR static constexpr UIntType QNAN_MASK = + UP::ENCODING == FPEncoding::X86_ExtendedPrecision + ? bit_at(SIG_BITS - 1) | bit_at(SIG_BITS - 2) // 0b1100... + : bit_at(SIG_BITS - 1); // 0b1000... + + LIBC_INLINE_VAR static constexpr UIntType SNAN_MASK = + UP::ENCODING == FPEncoding::X86_ExtendedPrecision + ? bit_at(SIG_BITS - 1) | bit_at(SIG_BITS - 3) // 0b1010... + : bit_at(SIG_BITS - 2); // 0b0100... + + // The number of bits after the decimal dot when the number if in normal form. + LIBC_INLINE_VAR static constexpr int FRACTION_BITS = + UP::ENCODING == FPEncoding::X86_ExtendedPrecision ? SIG_BITS - 1 + : SIG_BITS; + +public: + // Public facing API to keep the change local to this file. + using BitsType = UIntType; + + LIBC_INLINE_VAR static constexpr uint32_t BIT_WIDTH = TOTAL_BITS; + LIBC_INLINE_VAR static constexpr uint32_t MANTISSA_WIDTH = FRACTION_BITS; + LIBC_INLINE_VAR static constexpr uint32_t MANTISSA_PRECISION = + MANTISSA_WIDTH + 1; + LIBC_INLINE_VAR static constexpr BitsType MANTISSA_MASK = + mask_trailing_ones(); + LIBC_INLINE_VAR static constexpr uint32_t EXPONENT_WIDTH = EXP_BITS; + LIBC_INLINE_VAR static constexpr uint32_t EXPONENT_BIAS = + static_cast(EXP_BIAS); + LIBC_INLINE_VAR static constexpr BitsType SIGN_MASK = SIGN_MASK_; + LIBC_INLINE_VAR static constexpr BitsType EXPONENT_MASK = EXP_MASK; + LIBC_INLINE_VAR static constexpr BitsType EXP_MANT_MASK = EXP_MASK | SIG_MASK; // If a number x is a NAN, then it is a quiet NAN if: // QuietNaNMask & bits(x) != 0 // Else, it is a signalling NAN. - static constexpr BitsType QUIET_NAN_MASK = BitsType(1) - << (MANTISSA_WIDTH - 1); + static constexpr BitsType QUIET_NAN_MASK = QNAN_MASK; }; -// Properties for numbers represented in 128 bits long double on non x86 -// platform. -template <> struct FPProperties { - typedef UInt128 BitsType; +} // namespace internal - static constexpr uint32_t BIT_WIDTH = sizeof(BitsType) << 3; +template +struct FPProperties : public internal::FPCommonProperties {}; - static constexpr uint32_t MANTISSA_WIDTH = 112; - static constexpr uint32_t MANTISSA_PRECISION = MANTISSA_WIDTH + 1; - static constexpr uint32_t EXPONENT_WIDTH = 15; - static constexpr BitsType MANTISSA_MASK = (BitsType(1) << MANTISSA_WIDTH) - 1; - static constexpr BitsType SIGN_MASK = BitsType(1) - << (EXPONENT_WIDTH + MANTISSA_WIDTH); - static constexpr BitsType EXPONENT_MASK = ~(SIGN_MASK | MANTISSA_MASK); - static constexpr uint32_t EXPONENT_BIAS = 16383; +// ---------------- +// Work In Progress +// ---------------- +// The 'FPProperties' template specializations below are being slowly replaced +// with properties from 'FPCommonProperties' above. Once specializations are +// empty, 'FPProperties' declaration can be fully replace with +// 'FPCommonProperties' implementation. - static constexpr BitsType EXP_MANT_MASK = MANTISSA_MASK | EXPONENT_MASK; - static_assert(EXP_MANT_MASK == ~SIGN_MASK, - "Exponent and mantissa masks are not as expected."); - - // If a number x is a NAN, then it is a quiet NAN if: - // QuietNaNMask & bits(x) != 0 - // Else, it is a signalling NAN. - static constexpr BitsType QUIET_NAN_MASK = BitsType(1) - << (MANTISSA_WIDTH - 1); +// Properties for numbers represented in 80 bits long double on non-Windows x86 +// platforms. +template <> +struct FPProperties + : public internal::FPCommonProperties { + // The x86 80 bit float represents the leading digit of the mantissa + // explicitly. This is the mask for that bit. + static constexpr BitsType EXPLICIT_BIT_MASK = (BitsType(1) << MANTISSA_WIDTH); }; //----------------------------------------------------------------------------- diff --git a/libc/src/__support/FPUtil/Hypot.h b/libc/src/__support/FPUtil/Hypot.h index 8a6eb4b920ac..42d9e1b3f8ce 100644 --- a/libc/src/__support/FPUtil/Hypot.h +++ b/libc/src/__support/FPUtil/Hypot.h @@ -120,8 +120,8 @@ LIBC_INLINE T hypot(T x, T y) { return y; } - uint16_t x_exp = x_bits.get_unbiased_exponent(); - uint16_t y_exp = y_bits.get_unbiased_exponent(); + uint16_t x_exp = x_bits.get_biased_exponent(); + uint16_t y_exp = y_bits.get_biased_exponent(); uint16_t exp_diff = (x_exp > y_exp) ? (x_exp - y_exp) : (y_exp - x_exp); if ((exp_diff >= MantissaWidth::VALUE + 2) || (x == 0) || (y == 0)) { @@ -178,7 +178,7 @@ LIBC_INLINE T hypot(T x, T y) { // But before that, remember to store the losing bits to sticky. // The shift length is for a^2 and b^2, so it's double of the exponent // difference between a and b. - uint16_t shift_length = 2 * (a_exp - b_exp); + uint16_t shift_length = static_cast(2 * (a_exp - b_exp)); sticky_bits = ((b_mant_sq & ((DUIntType(1) << shift_length) - DUIntType(1))) != DUIntType(0)); diff --git a/libc/src/__support/FPUtil/ManipulationFunctions.h b/libc/src/__support/FPUtil/ManipulationFunctions.h index 9d3fd075be47..08adb074b121 100644 --- a/libc/src/__support/FPUtil/ManipulationFunctions.h +++ b/libc/src/__support/FPUtil/ManipulationFunctions.h @@ -144,10 +144,11 @@ LIBC_INLINE T ldexp(T x, int exp) { return normal; } -template < - typename T, typename U, - cpp::enable_if_t && cpp::is_floating_point_v, - int> = 0> +template && + cpp::is_floating_point_v && + (sizeof(T) <= sizeof(U)), + int> = 0> LIBC_INLINE T nextafter(T from, U to) { FPBits from_bits(from); if (from_bits.is_nan()) @@ -157,6 +158,9 @@ LIBC_INLINE T nextafter(T from, U to) { if (to_bits.is_nan()) return static_cast(to); + // NOTE: This would work only if `U` has a greater or equal precision than + // `T`. Otherwise `from` could loose its precision and the following statement + // could incorrectly evaluate to `true`. if (static_cast(from) == to) return static_cast(to); diff --git a/libc/src/__support/FPUtil/NormalFloat.h b/libc/src/__support/FPUtil/NormalFloat.h index d59de14fb695..d3236316a879 100644 --- a/libc/src/__support/FPUtil/NormalFloat.h +++ b/libc/src/__support/FPUtil/NormalFloat.h @@ -111,7 +111,7 @@ template struct NormalFloat { const UIntType shift_out_mask = (UIntType(1) << shift) - 1; const UIntType shift_out_value = mantissa & shift_out_mask; const UIntType halfway_value = UIntType(1) << (shift - 1); - result.set_unbiased_exponent(0); + result.set_biased_exponent(0); result.set_mantissa(mantissa >> shift); UIntType new_mantissa = result.get_mantissa(); if (shift_out_value > halfway_value) { @@ -126,14 +126,14 @@ template struct NormalFloat { // mantissa was all ones (0b111..11). For such a case, we will carry // the overflow into the exponent. if (new_mantissa == ONE) - result.set_unbiased_exponent(1); + result.set_biased_exponent(1); return T(result); } else { return T(result); } } - result.set_unbiased_exponent(exponent + FPBits::EXPONENT_BIAS); + result.set_biased_exponent(exponent + FPBits::EXPONENT_BIAS); result.set_mantissa(mantissa); return T(result); } @@ -151,12 +151,12 @@ private: } // Normalize subnormal numbers. - if (bits.get_unbiased_exponent() == 0) { + if (bits.get_biased_exponent() == 0) { unsigned shift = evaluate_normalization_shift(bits.get_mantissa()); mantissa = UIntType(bits.get_mantissa()) << shift; exponent = 1 - FPBits::EXPONENT_BIAS - shift; } else { - exponent = bits.get_unbiased_exponent() - FPBits::EXPONENT_BIAS; + exponent = bits.get_biased_exponent() - FPBits::EXPONENT_BIAS; mantissa = ONE | bits.get_mantissa(); } } @@ -184,7 +184,7 @@ NormalFloat::init_from_bits(FPBits bits) { return; } - if (bits.get_unbiased_exponent() == 0) { + if (bits.get_biased_exponent() == 0) { if (bits.get_implicit_bit() == 0) { // Since we ignore zero value, the mantissa in this case is non-zero. int normalization_shift = @@ -201,7 +201,7 @@ NormalFloat::init_from_bits(FPBits bits) { exponent = 0; mantissa = 0; } else { - exponent = bits.get_unbiased_exponent() - 16383; + exponent = bits.get_biased_exponent() - 16383; mantissa = ONE | bits.get_mantissa(); } } @@ -228,7 +228,7 @@ template <> LIBC_INLINE NormalFloat::operator long double() const { const UIntType shift_out_mask = (UIntType(1) << shift) - 1; const UIntType shift_out_value = mantissa & shift_out_mask; const UIntType halfway_value = UIntType(1) << (shift - 1); - result.set_unbiased_exponent(0); + result.set_biased_exponent(0); result.set_mantissa(mantissa >> shift); UIntType new_mantissa = result.get_mantissa(); if (shift_out_value > halfway_value) { @@ -243,7 +243,7 @@ template <> LIBC_INLINE NormalFloat::operator long double() const { // mantissa was all ones (0b111..11). For such a case, we will carry // the overflow into the exponent and set the implicit bit to 1. if (new_mantissa == ONE) { - result.set_unbiased_exponent(1); + result.set_biased_exponent(1); result.set_implicit_bit(1); } else { result.set_implicit_bit(0); @@ -254,7 +254,7 @@ template <> LIBC_INLINE NormalFloat::operator long double() const { } } - result.set_unbiased_exponent(biased_exponent); + result.set_biased_exponent(biased_exponent); result.set_mantissa(mantissa); result.set_implicit_bit(1); return static_cast(result); diff --git a/libc/src/__support/FPUtil/dyadic_float.h b/libc/src/__support/FPUtil/dyadic_float.h index b7920943804e..5f0d8f49ccf6 100644 --- a/libc/src/__support/FPUtil/dyadic_float.h +++ b/libc/src/__support/FPUtil/dyadic_float.h @@ -94,7 +94,7 @@ template struct DyadicFloat { return 0.0; // Assume that it is normalized, and output is also normal. - constexpr size_t PRECISION = FloatProperties::MANTISSA_WIDTH + 1; + constexpr uint32_t PRECISION = FloatProperties::MANTISSA_PRECISION; using output_bits_t = typename FPBits::UIntType; int exp_hi = exponent + static_cast((Bits - 1) + @@ -110,12 +110,12 @@ template struct DyadicFloat { exp_hi = FloatProperties::EXPONENT_BIAS; } - int exp_lo = exp_hi - PRECISION - 1; + int exp_lo = exp_hi - static_cast(PRECISION) - 1; MantissaType m_hi(mantissa >> shift); T d_hi = FPBits::create_value(sign, exp_hi, - output_bits_t(m_hi) & + static_cast(m_hi) & FloatProperties::MANTISSA_MASK) .get_val(); diff --git a/libc/src/__support/FPUtil/fpbits_str.h b/libc/src/__support/FPUtil/fpbits_str.h index 4dec85ac2cc8..5d0bb6cf1ac4 100644 --- a/libc/src/__support/FPUtil/fpbits_str.h +++ b/libc/src/__support/FPUtil/fpbits_str.h @@ -53,7 +53,7 @@ template LIBC_INLINE cpp::string str(fputil::FPBits x) { s += sign_char(x.get_sign()); s += ", E: "; - const details::ZeroPaddedHexFmt exponent(x.get_unbiased_exponent()); + const details::ZeroPaddedHexFmt exponent(x.get_biased_exponent()); s += exponent.view(); if constexpr (cpp::is_same_v && diff --git a/libc/src/__support/FPUtil/generic/FMA.h b/libc/src/__support/FPUtil/generic/FMA.h index 4a825f2f4942..61a1401c30e8 100644 --- a/libc/src/__support/FPUtil/generic/FMA.h +++ b/libc/src/__support/FPUtil/generic/FMA.h @@ -58,7 +58,7 @@ template <> LIBC_INLINE float fma(float x, float y, float z) { // bit of sum, so that the sticky bits used when rounding sum to float are // correct (when it matters). fputil::FPBits t( - (bit_prod.get_unbiased_exponent() >= bitz.get_unbiased_exponent()) + (bit_prod.get_biased_exponent() >= bitz.get_biased_exponent()) ? ((double(bit_sum) - double(bit_prod)) - double(bitz)) : ((double(bit_sum) - double(bitz)) - double(bit_prod))); @@ -106,15 +106,15 @@ template <> LIBC_INLINE double fma(double x, double y, double z) { int z_exp = 0; // Normalize denormal inputs. - if (LIBC_UNLIKELY(FPBits(x).get_unbiased_exponent() == 0)) { + if (LIBC_UNLIKELY(FPBits(x).get_biased_exponent() == 0)) { x_exp -= 52; x *= 0x1.0p+52; } - if (LIBC_UNLIKELY(FPBits(y).get_unbiased_exponent() == 0)) { + if (LIBC_UNLIKELY(FPBits(y).get_biased_exponent() == 0)) { y_exp -= 52; y *= 0x1.0p+52; } - if (LIBC_UNLIKELY(FPBits(z).get_unbiased_exponent() == 0)) { + if (LIBC_UNLIKELY(FPBits(z).get_biased_exponent() == 0)) { z_exp -= 52; z *= 0x1.0p+52; } @@ -124,9 +124,9 @@ template <> LIBC_INLINE double fma(double x, double y, double z) { bool y_sign = y_bits.get_sign(); bool z_sign = z_bits.get_sign(); bool prod_sign = x_sign != y_sign; - x_exp += x_bits.get_unbiased_exponent(); - y_exp += y_bits.get_unbiased_exponent(); - z_exp += z_bits.get_unbiased_exponent(); + x_exp += x_bits.get_biased_exponent(); + y_exp += y_bits.get_biased_exponent(); + z_exp += z_bits.get_biased_exponent(); if (LIBC_UNLIKELY(x_exp == FPBits::MAX_EXPONENT || y_exp == FPBits::MAX_EXPONENT || diff --git a/libc/src/__support/FPUtil/generic/FMod.h b/libc/src/__support/FPUtil/generic/FMod.h index 0e71b039d5c0..7502660c88a1 100644 --- a/libc/src/__support/FPUtil/generic/FMod.h +++ b/libc/src/__support/FPUtil/generic/FMod.h @@ -233,8 +233,8 @@ private: return FPB(FPB::zero()); // |x|=|y| return 0.0 } - int e_x = sx.get_unbiased_exponent(); - int e_y = sy.get_unbiased_exponent(); + int e_x = sx.get_biased_exponent(); + int e_y = sy.get_biased_exponent(); // Most common case where |y| is "very normal" and |x/y| < 2^EXPONENT_WIDTH if (LIBC_LIKELY(e_y > int(FPB::FloatProp::MANTISSA_WIDTH) && diff --git a/libc/src/__support/FPUtil/generic/sqrt.h b/libc/src/__support/FPUtil/generic/sqrt.h index 6ae2171bacf7..5bde9589fdc0 100644 --- a/libc/src/__support/FPUtil/generic/sqrt.h +++ b/libc/src/__support/FPUtil/generic/sqrt.h @@ -97,7 +97,7 @@ LIBC_INLINE cpp::enable_if_t, T> sqrt(T x) { UIntType x_mant = bits.get_mantissa(); // Step 1a: Normalize denormal input and append hidden bit to the mantissa - if (bits.get_unbiased_exponent() == 0) { + if (bits.get_biased_exponent() == 0) { ++x_exp; // let x_exp be the correct exponent of ONE bit. internal::normalize(x_exp, x_mant); } else { diff --git a/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h b/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h index 0e7907e82943..2f25be54e0bc 100644 --- a/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h +++ b/libc/src/__support/FPUtil/generic/sqrt_80_bit_long_double.h @@ -65,7 +65,7 @@ LIBC_INLINE long double sqrt(long double x) { // Step 1a: Normalize denormal input if (bits.get_implicit_bit()) { x_mant |= ONE; - } else if (bits.get_unbiased_exponent() == 0) { + } else if (bits.get_biased_exponent() == 0) { normalize(x_exp, x_mant); } @@ -128,7 +128,7 @@ LIBC_INLINE long double sqrt(long double x) { // Extract output FPBits out(0.0L); - out.set_unbiased_exponent(x_exp); + out.set_biased_exponent(x_exp); out.set_implicit_bit(1); out.set_mantissa((y & (ONE - 1))); diff --git a/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h b/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h index 2f07ff4a2c3e..bbc30ff7a376 100644 --- a/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h +++ b/libc/src/__support/FPUtil/x86_64/LongDoubleBits.h @@ -71,7 +71,7 @@ template <> struct FPBits { return bits & (FloatProp::MANTISSA_MASK | FloatProp::EXPLICIT_BIT_MASK); } - LIBC_INLINE constexpr void set_unbiased_exponent(UIntType expVal) { + LIBC_INLINE constexpr void set_biased_exponent(UIntType expVal) { expVal = (expVal << (FloatProp::BIT_WIDTH - 1 - FloatProp::EXPONENT_WIDTH)) & FloatProp::EXPONENT_MASK; @@ -79,7 +79,7 @@ template <> struct FPBits { bits |= expVal; } - LIBC_INLINE constexpr uint16_t get_unbiased_exponent() const { + LIBC_INLINE constexpr uint16_t get_biased_exponent() const { return uint16_t((bits & FloatProp::EXPONENT_MASK) >> (FloatProp::BIT_WIDTH - 1 - FloatProp::EXPONENT_WIDTH)); } @@ -137,7 +137,7 @@ template <> struct FPBits { } LIBC_INLINE constexpr int get_exponent() const { - return int(get_unbiased_exponent()) - EXPONENT_BIAS; + return int(get_biased_exponent()) - EXPONENT_BIAS; } // If the number is subnormal, the exponent is treated as if it were the @@ -147,38 +147,38 @@ template <> struct FPBits { // will give a slightly incorrect result. Additionally, zero has an exponent // of zero, and that should actually be treated as zero. LIBC_INLINE constexpr int get_explicit_exponent() const { - const int unbiased_exp = int(get_unbiased_exponent()); + const int biased_exp = int(get_biased_exponent()); if (is_zero()) { return 0; - } else if (unbiased_exp == 0) { + } else if (biased_exp == 0) { return 1 - EXPONENT_BIAS; } else { - return unbiased_exp - EXPONENT_BIAS; + return biased_exp - EXPONENT_BIAS; } } LIBC_INLINE constexpr bool is_zero() const { - return get_unbiased_exponent() == 0 && get_mantissa() == 0 && + return get_biased_exponent() == 0 && get_mantissa() == 0 && get_implicit_bit() == 0; } LIBC_INLINE constexpr bool is_inf() const { - return get_unbiased_exponent() == MAX_EXPONENT && get_mantissa() == 0 && + return get_biased_exponent() == MAX_EXPONENT && get_mantissa() == 0 && get_implicit_bit() == 1; } LIBC_INLINE constexpr bool is_nan() const { - if (get_unbiased_exponent() == MAX_EXPONENT) { + if (get_biased_exponent() == MAX_EXPONENT) { return (get_implicit_bit() == 0) || get_mantissa() != 0; - } else if (get_unbiased_exponent() != 0) { + } else if (get_biased_exponent() != 0) { return get_implicit_bit() == 0; } return false; } LIBC_INLINE constexpr bool is_inf_or_nan() const { - return (get_unbiased_exponent() == MAX_EXPONENT) || - (get_unbiased_exponent() != 0 && get_implicit_bit() == 0); + return (get_biased_exponent() == MAX_EXPONENT) || + (get_biased_exponent() != 0 && get_implicit_bit() == 0); } // Methods below this are used by tests. @@ -189,7 +189,7 @@ template <> struct FPBits { LIBC_INLINE static constexpr long double inf(bool sign = false) { FPBits bits(0.0l); - bits.set_unbiased_exponent(MAX_EXPONENT); + bits.set_biased_exponent(MAX_EXPONENT); bits.set_implicit_bit(1); if (sign) { bits.set_sign(true); @@ -201,7 +201,7 @@ template <> struct FPBits { LIBC_INLINE static constexpr long double build_nan(UIntType v) { FPBits bits(0.0l); - bits.set_unbiased_exponent(MAX_EXPONENT); + bits.set_biased_exponent(MAX_EXPONENT); bits.set_implicit_bit(1); bits.set_mantissa(v); return bits; @@ -228,10 +228,10 @@ template <> struct FPBits { } LIBC_INLINE static constexpr FPBits - create_value(bool sign, UIntType unbiased_exp, UIntType mantissa) { + create_value(bool sign, UIntType biased_exp, UIntType mantissa) { FPBits result; result.set_sign(sign); - result.set_unbiased_exponent(unbiased_exp); + result.set_biased_exponent(biased_exp); result.set_mantissa(mantissa); return result; } diff --git a/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h b/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h index 4508671b47ee..5e32f766ad58 100644 --- a/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h +++ b/libc/src/__support/FPUtil/x86_64/NextAfterLongDouble.h @@ -39,8 +39,8 @@ LIBC_INLINE long double nextafter(long double from, long double to) { // Convert pseudo subnormal number to normal number. if (from_bits.get_implicit_bit() == 1 && - from_bits.get_unbiased_exponent() == 0) { - from_bits.set_unbiased_exponent(1); + from_bits.get_biased_exponent() == 0) { + from_bits.set_biased_exponent(1); } using UIntType = FPBits::UIntType; @@ -59,7 +59,7 @@ LIBC_INLINE long double nextafter(long double from, long double to) { // Incrementing exponent might overflow the value to infinity, // which is what is expected. Since NaNs are handling separately, // it will never overflow "beyond" infinity. - from_bits.set_unbiased_exponent(from_bits.get_unbiased_exponent() + 1); + from_bits.set_biased_exponent(from_bits.get_biased_exponent() + 1); if (from_bits.is_inf()) raise_except_if_required(FE_OVERFLOW | FE_INEXACT); return from_bits; @@ -75,7 +75,7 @@ LIBC_INLINE long double nextafter(long double from, long double to) { from_bits.set_mantissa(MANTISSA_MASK); // from == 0 is handled separately so decrementing the exponent will not // lead to underflow. - from_bits.set_unbiased_exponent(from_bits.get_unbiased_exponent() - 1); + from_bits.set_biased_exponent(from_bits.get_biased_exponent() - 1); return from_bits; } else { --int_val; @@ -94,7 +94,7 @@ LIBC_INLINE long double nextafter(long double from, long double to) { from_bits.set_mantissa(MANTISSA_MASK); // from == 0 is handled separately so decrementing the exponent will not // lead to underflow. - from_bits.set_unbiased_exponent(from_bits.get_unbiased_exponent() - 1); + from_bits.set_biased_exponent(from_bits.get_biased_exponent() - 1); return from_bits; } else { --int_val; @@ -107,7 +107,7 @@ LIBC_INLINE long double nextafter(long double from, long double to) { // Incrementing exponent might overflow the value to infinity, // which is what is expected. Since NaNs are handling separately, // it will never overflow "beyond" infinity. - from_bits.set_unbiased_exponent(from_bits.get_unbiased_exponent() + 1); + from_bits.set_biased_exponent(from_bits.get_biased_exponent() + 1); if (from_bits.is_inf()) raise_except_if_required(FE_OVERFLOW | FE_INEXACT); return from_bits; diff --git a/libc/src/__support/UInt.h b/libc/src/__support/UInt.h index 3bec2e3a4713..f72b995f8788 100644 --- a/libc/src/__support/UInt.h +++ b/libc/src/__support/UInt.h @@ -103,13 +103,20 @@ template struct BigInt { val[i] = words[i]; } - template && - sizeof(T) <= 16 && - !cpp::is_same_v>> - LIBC_INLINE constexpr explicit operator T() const { - if constexpr (sizeof(T) <= 8) - return static_cast(val[0]); + template LIBC_INLINE constexpr explicit operator T() const { + return to(); + } + template + LIBC_INLINE constexpr cpp::enable_if_t< + cpp::is_integral_v && sizeof(T) <= 8 && !cpp::is_same_v, T> + to() const { + return static_cast(val[0]); + } + template + LIBC_INLINE constexpr cpp::enable_if_t< + cpp::is_integral_v && sizeof(T) == 16, T> + to() const { // T is 128-bit. T lo = static_cast(val[0]); @@ -121,7 +128,6 @@ template struct BigInt { return lo; } } else { - // TODO: silence shift warning return static_cast((static_cast(val[1]) << 64) + lo); } } diff --git a/libc/src/__support/str_to_float.h b/libc/src/__support/str_to_float.h index ad73e93f6faa..7ec6b9947ad2 100644 --- a/libc/src/__support/str_to_float.h +++ b/libc/src/__support/str_to_float.h @@ -9,6 +9,7 @@ #ifndef LLVM_LIBC_SRC___SUPPORT_STR_TO_FLOAT_H #define LLVM_LIBC_SRC___SUPPORT_STR_TO_FLOAT_H +#include "src/__support/CPP/bit.h" #include "src/__support/CPP/limits.h" #include "src/__support/CPP/optional.h" #include "src/__support/FPUtil/FEnvImpl.h" @@ -37,45 +38,6 @@ template struct FloatConvertReturn { int error = 0; }; -template LIBC_INLINE uint32_t leading_zeroes(T inputNumber) { - constexpr uint32_t BITS_IN_T = sizeof(T) * 8; - if (inputNumber == 0) { - return BITS_IN_T; - } - uint32_t cur_guess = BITS_IN_T / 2; - uint32_t range_size = BITS_IN_T / 2; - // while either shifting by curGuess does not get rid of all of the bits or - // shifting by one less also gets rid of all of the bits then we have not - // found the first bit. - while (((inputNumber >> cur_guess) > 0) || - ((inputNumber >> (cur_guess - 1)) == 0)) { - // Binary search for the first set bit - range_size /= 2; - if (range_size == 0) { - break; - } - if ((inputNumber >> cur_guess) > 0) { - cur_guess += range_size; - } else { - cur_guess -= range_size; - } - } - if (inputNumber >> cur_guess > 0) { - cur_guess++; - } - return BITS_IN_T - cur_guess; -} - -template <> -LIBC_INLINE uint32_t leading_zeroes(uint32_t inputNumber) { - return cpp::countl_zero(inputNumber); -} - -template <> -LIBC_INLINE uint32_t leading_zeroes(uint64_t inputNumber) { - return cpp::countl_zero(inputNumber); -} - LIBC_INLINE uint64_t low64(const UInt128 &num) { return static_cast(num & 0xffffffffffffffff); } @@ -92,7 +54,7 @@ template LIBC_INLINE void set_implicit_bit(fputil::FPBits &) { template <> LIBC_INLINE void set_implicit_bit(fputil::FPBits &result) { - result.set_implicit_bit(result.get_unbiased_exponent() != 0); + result.set_implicit_bit(result.get_biased_exponent() != 0); } #endif @@ -108,10 +70,11 @@ template LIBC_INLINE cpp::optional> eisel_lemire(ExpandedFloat init_num, RoundDirection round = RoundDirection::Nearest) { + using FPBits = typename fputil::FPBits; + using FloatProp = typename FPBits::FloatProp; + using UIntType = typename FPBits::UIntType; - using BitsType = typename fputil::FPBits::UIntType; - - BitsType mantissa = init_num.mantissa; + UIntType mantissa = init_num.mantissa; int32_t exp10 = init_num.exponent; constexpr uint32_t BITS_IN_MANTISSA = sizeof(mantissa) * 8; @@ -128,12 +91,11 @@ eisel_lemire(ExpandedFloat init_num, } // Normalization - uint32_t clz = leading_zeroes(mantissa); + uint32_t clz = cpp::countl_zero(mantissa); mantissa <<= clz; uint32_t exp2 = static_cast(exp10_to_exp2(exp10)) + - BITS_IN_MANTISSA + fputil::FloatProperties::EXPONENT_BIAS - - clz; + BITS_IN_MANTISSA + FloatProp::EXPONENT_BIAS - clz; // Multiplication const uint64_t *power_of_ten = @@ -150,9 +112,7 @@ eisel_lemire(ExpandedFloat init_num, // accuracy, and the most significant bit is ignored.) = 9 bits. Similarly, // it's 6 bits for floats in this case. const uint64_t halfway_constant = - (uint64_t(1) << (BITS_IN_MANTISSA - - fputil::FloatProperties::MANTISSA_WIDTH - 3)) - - 1; + (uint64_t(1) << (BITS_IN_MANTISSA - (FloatProp::MANTISSA_WIDTH + 3))) - 1; if ((high64(first_approx) & halfway_constant) == halfway_constant && low64(first_approx) + mantissa < mantissa) { UInt128 low_bits = @@ -171,12 +131,11 @@ eisel_lemire(ExpandedFloat init_num, } // Shifting to 54 bits for doubles and 25 bits for floats - BitsType msb = - static_cast(high64(final_approx) >> (BITS_IN_MANTISSA - 1)); - BitsType final_mantissa = - static_cast(high64(final_approx) >> - (msb + BITS_IN_MANTISSA - - (fputil::FloatProperties::MANTISSA_WIDTH + 3))); + UIntType msb = + static_cast(high64(final_approx) >> (BITS_IN_MANTISSA - 1)); + UIntType final_mantissa = static_cast( + high64(final_approx) >> + (msb + BITS_IN_MANTISSA - (FloatProp::MANTISSA_WIDTH + 3))); exp2 -= static_cast(1 ^ msb); // same as !msb if (round == RoundDirection::Nearest) { @@ -202,15 +161,14 @@ eisel_lemire(ExpandedFloat init_num, // From 54 to 53 bits for doubles and 25 to 24 bits for floats final_mantissa >>= 1; - if ((final_mantissa >> (fputil::FloatProperties::MANTISSA_WIDTH + 1)) > - 0) { + if ((final_mantissa >> (FloatProp::MANTISSA_WIDTH + 1)) > 0) { final_mantissa >>= 1; ++exp2; } // The if block is equivalent to (but has fewer branches than): // if exp2 <= 0 || exp2 >= 0x7FF { etc } - if (exp2 - 1 >= (1 << fputil::FloatProperties::EXPONENT_WIDTH) - 2) { + if (exp2 - 1 >= (1 << FloatProp::EXPONENT_WIDTH) - 2) { return cpp::nullopt; } @@ -225,9 +183,11 @@ template <> LIBC_INLINE cpp::optional> eisel_lemire(ExpandedFloat init_num, RoundDirection round) { - using BitsType = typename fputil::FPBits::UIntType; + using FPBits = typename fputil::FPBits; + using FloatProp = typename FPBits::FloatProp; + using UIntType = typename FPBits::UIntType; - BitsType mantissa = init_num.mantissa; + UIntType mantissa = init_num.mantissa; int32_t exp10 = init_num.exponent; constexpr uint32_t BITS_IN_MANTISSA = sizeof(mantissa) * 8; @@ -248,12 +208,11 @@ eisel_lemire(ExpandedFloat init_num, } // Normalization - uint32_t clz = leading_zeroes(mantissa); + uint32_t clz = cpp::countl_zero(mantissa); mantissa <<= clz; uint32_t exp2 = static_cast(exp10_to_exp2(exp10)) + - BITS_IN_MANTISSA + - fputil::FloatProperties::EXPONENT_BIAS - clz; + BITS_IN_MANTISSA + FloatProp::EXPONENT_BIAS - clz; // Multiplication const uint64_t *power_of_ten = @@ -290,10 +249,7 @@ eisel_lemire(ExpandedFloat init_num, // accuracy, and the most significant bit is ignored.) = 61 bits. Similarly, // it's 12 bits for 128 bit floats in this case. constexpr UInt128 HALFWAY_CONSTANT = - (UInt128(1) << (BITS_IN_MANTISSA - - fputil::FloatProperties::MANTISSA_WIDTH - - 3)) - - 1; + (UInt128(1) << (BITS_IN_MANTISSA - (FloatProp::MANTISSA_WIDTH + 3))) - 1; if ((final_approx_upper & HALFWAY_CONSTANT) == HALFWAY_CONSTANT && final_approx_lower + mantissa < mantissa) { @@ -303,10 +259,9 @@ eisel_lemire(ExpandedFloat init_num, // Shifting to 65 bits for 80 bit floats and 113 bits for 128 bit floats uint32_t msb = static_cast(final_approx_upper >> (BITS_IN_MANTISSA - 1)); - BitsType final_mantissa = + UIntType final_mantissa = final_approx_upper >> - (msb + BITS_IN_MANTISSA - - (fputil::FloatProperties::MANTISSA_WIDTH + 3)); + (msb + BITS_IN_MANTISSA - (FloatProp::MANTISSA_WIDTH + 3)); exp2 -= static_cast(1 ^ msb); // same as !msb if (round == RoundDirection::Nearest) { @@ -331,16 +286,14 @@ eisel_lemire(ExpandedFloat init_num, // From 65 to 64 bits for 80 bit floats and 113 to 112 bits for 128 bit // floats final_mantissa >>= 1; - if ((final_mantissa >> - (fputil::FloatProperties::MANTISSA_WIDTH + 1)) > 0) { + if ((final_mantissa >> (FloatProp::MANTISSA_WIDTH + 1)) > 0) { final_mantissa >>= 1; ++exp2; } // The if block is equivalent to (but has fewer branches than): // if exp2 <= 0 || exp2 >= MANTISSA_MAX { etc } - if (exp2 - 1 >= - (1 << fputil::FloatProperties::EXPONENT_WIDTH) - 2) { + if (exp2 - 1 >= (1 << FloatProp::EXPONENT_WIDTH) - 2) { return cpp::nullopt; } @@ -368,6 +321,9 @@ template LIBC_INLINE FloatConvertReturn simple_decimal_conversion(const char *__restrict numStart, RoundDirection round = RoundDirection::Nearest) { + using FPBits = typename fputil::FPBits; + using FloatProp = typename FPBits::FloatProp; + using UIntType = typename FPBits::UIntType; int32_t exp2 = 0; HighPrecisionDecimal hpd = HighPrecisionDecimal(numStart); @@ -383,16 +339,16 @@ simple_decimal_conversion(const char *__restrict numStart, // float, return inf. if (hpd.get_decimal_point() > 0 && exp10_to_exp2(hpd.get_decimal_point() - 1) > - static_cast(fputil::FloatProperties::EXPONENT_BIAS)) { - output.num = {0, fputil::FPBits::MAX_EXPONENT}; + static_cast(FloatProp::EXPONENT_BIAS)) { + output.num = {0, FPBits::MAX_EXPONENT}; output.error = ERANGE; return output; } // If the exponent is too small even for a subnormal, return 0. if (hpd.get_decimal_point() < 0 && exp10_to_exp2(-hpd.get_decimal_point()) > - static_cast(fputil::FloatProperties::EXPONENT_BIAS + - fputil::FloatProperties::MANTISSA_WIDTH)) { + static_cast(FloatProp::EXPONENT_BIAS + + FloatProp::MANTISSA_WIDTH)) { output.num = {0, 0}; output.error = ERANGE; return output; @@ -431,19 +387,18 @@ simple_decimal_conversion(const char *__restrict numStart, hpd.shift(1); // Get the biased exponent - exp2 += fputil::FloatProperties::EXPONENT_BIAS; + exp2 += FloatProp::EXPONENT_BIAS; // Handle the exponent being too large (and return inf). - if (exp2 >= fputil::FPBits::MAX_EXPONENT) { - output.num = {0, fputil::FPBits::MAX_EXPONENT}; + if (exp2 >= FPBits::MAX_EXPONENT) { + output.num = {0, FPBits::MAX_EXPONENT}; output.error = ERANGE; return output; } // Shift left to fill the mantissa - hpd.shift(fputil::FloatProperties::MANTISSA_WIDTH); - typename fputil::FPBits::UIntType final_mantissa = - hpd.round_to_integer_type::UIntType>(); + hpd.shift(FloatProp::MANTISSA_WIDTH); + UIntType final_mantissa = hpd.round_to_integer_type(); // Handle subnormals if (exp2 <= 0) { @@ -455,25 +410,23 @@ simple_decimal_conversion(const char *__restrict numStart, // Shift right one more time to compensate for the left shift to get it // between 1 and 2. hpd.shift(-1); - final_mantissa = - hpd.round_to_integer_type::UIntType>(round); + final_mantissa = hpd.round_to_integer_type(round); // Check if by shifting right we've caused this to round to a normal number. - if ((final_mantissa >> fputil::FloatProperties::MANTISSA_WIDTH) != 0) { + if ((final_mantissa >> FloatProp::MANTISSA_WIDTH) != 0) { ++exp2; } } // Check if rounding added a bit, and shift down if that's the case. - if (final_mantissa == typename fputil::FPBits::UIntType(2) - << fputil::FloatProperties::MANTISSA_WIDTH) { + if (final_mantissa == UIntType(2) << FloatProp::MANTISSA_WIDTH) { final_mantissa >>= 1; ++exp2; // Check if this rounding causes exp2 to go out of range and make the result // INF. If this is the case, then finalMantissa and exp2 are already the // correct values for an INF result. - if (exp2 >= fputil::FPBits::MAX_EXPONENT) { + if (exp2 >= FPBits::MAX_EXPONENT) { output.error = ERANGE; } } @@ -563,18 +516,20 @@ template LIBC_INLINE cpp::optional> clinger_fast_path(ExpandedFloat init_num, RoundDirection round = RoundDirection::Nearest) { + using FPBits = typename fputil::FPBits; + using FloatProp = typename FPBits::FloatProp; + using UIntType = typename FPBits::UIntType; - typename fputil::FPBits::UIntType mantissa = init_num.mantissa; + UIntType mantissa = init_num.mantissa; int32_t exp10 = init_num.exponent; - if (mantissa >> fputil::FloatProperties::MANTISSA_WIDTH > 0) { + if ((mantissa >> FloatProp::MANTISSA_WIDTH) > 0) { return cpp::nullopt; } - fputil::FPBits result; + FPBits result; T float_mantissa; - if constexpr (cpp::is_same_v::UIntType, - cpp::UInt<128>>) { + if constexpr (cpp::is_same_v>) { float_mantissa = static_cast(fputil::DyadicFloat<128>( false, 0, fputil::DyadicFloat<128>::MantissaType( @@ -584,7 +539,7 @@ clinger_fast_path(ExpandedFloat init_num, } if (exp10 == 0) { - result = fputil::FPBits(float_mantissa); + result = FPBits(float_mantissa); } if (exp10 > 0) { if (exp10 > ClingerConsts::EXACT_POWERS_OF_TEN + @@ -600,14 +555,14 @@ clinger_fast_path(ExpandedFloat init_num, if (float_mantissa > ClingerConsts::MAX_EXACT_INT) { return cpp::nullopt; } - result = fputil::FPBits(float_mantissa * - ClingerConsts::POWERS_OF_TEN_ARRAY[exp10]); + result = + FPBits(float_mantissa * ClingerConsts::POWERS_OF_TEN_ARRAY[exp10]); } else if (exp10 < 0) { if (-exp10 > ClingerConsts::EXACT_POWERS_OF_TEN) { return cpp::nullopt; } - result = fputil::FPBits(float_mantissa / - ClingerConsts::POWERS_OF_TEN_ARRAY[-exp10]); + result = + FPBits(float_mantissa / ClingerConsts::POWERS_OF_TEN_ARRAY[-exp10]); } // If the rounding mode is not nearest, then the sign of the number may affect @@ -615,15 +570,15 @@ clinger_fast_path(ExpandedFloat init_num, // calculation is redone with a negative result, and the rounding mode is used // to select the correct result. if (round != RoundDirection::Nearest) { - fputil::FPBits negative_result; + FPBits negative_result; // I'm 99% sure this will break under fast math optimizations. - negative_result = fputil::FPBits( - (-float_mantissa) * ClingerConsts::POWERS_OF_TEN_ARRAY[exp10]); + negative_result = FPBits((-float_mantissa) * + ClingerConsts::POWERS_OF_TEN_ARRAY[exp10]); // If the results are equal, then we don't need to use the rounding mode. if (T(result) != -T(negative_result)) { - fputil::FPBits lower_result; - fputil::FPBits higher_result; + FPBits lower_result; + FPBits higher_result; if (T(result) < -T(negative_result)) { lower_result = result; @@ -643,7 +598,7 @@ clinger_fast_path(ExpandedFloat init_num, ExpandedFloat output; output.mantissa = result.get_mantissa(); - output.exponent = result.get_unbiased_exponent(); + output.exponent = result.get_biased_exponent(); return output; } @@ -691,8 +646,10 @@ template LIBC_INLINE FloatConvertReturn decimal_exp_to_float(ExpandedFloat init_num, const char *__restrict numStart, bool truncated, RoundDirection round) { + using FPBits = typename fputil::FPBits; + using UIntType = typename FPBits::UIntType; - typename fputil::FPBits::UIntType mantissa = init_num.mantissa; + UIntType mantissa = init_num.mantissa; int32_t exp10 = init_num.exponent; FloatConvertReturn output; @@ -702,7 +659,7 @@ decimal_exp_to_float(ExpandedFloat init_num, const char *__restrict numStart, // float, return inf. These bounds are relatively loose, but are mostly // serving as a first pass. Some close numbers getting through is okay. if (exp10 > get_upper_bound()) { - output.num = {0, fputil::FPBits::MAX_EXPONENT}; + output.num = {0, FPBits::MAX_EXPONENT}; output.error = ERANGE; return output; } @@ -766,40 +723,39 @@ template LIBC_INLINE FloatConvertReturn binary_exp_to_float(ExpandedFloat init_num, bool truncated, RoundDirection round) { - using BitsType = typename fputil::FPBits::UIntType; + using FPBits = typename fputil::FPBits; + using FloatProp = typename FPBits::FloatProp; + using UIntType = typename FPBits::UIntType; - BitsType mantissa = init_num.mantissa; + UIntType mantissa = init_num.mantissa; int32_t exp2 = init_num.exponent; FloatConvertReturn output; // This is the number of leading zeroes a properly normalized float of type T // should have. - constexpr int32_t NUMBITS = sizeof(BitsType) * 8; - constexpr int32_t INF_EXP = - (1 << fputil::FloatProperties::EXPONENT_WIDTH) - 1; + constexpr int32_t NUMBITS = sizeof(UIntType) * 8; + constexpr int32_t INF_EXP = (1 << FloatProp::EXPONENT_WIDTH) - 1; - // Normalization step 1: Bring the leading bit to the highest bit of BitsType. - uint32_t amount_to_shift_left = leading_zeroes(mantissa); + // Normalization step 1: Bring the leading bit to the highest bit of UIntType. + uint32_t amount_to_shift_left = cpp::countl_zero(mantissa); mantissa <<= amount_to_shift_left; - // Keep exp2 representing the exponent of the lowest bit of BitsType. + // Keep exp2 representing the exponent of the lowest bit of UIntType. exp2 -= amount_to_shift_left; // biasedExponent represents the biased exponent of the most significant bit. - int32_t biased_exponent = - exp2 + NUMBITS + fputil::FPBits::EXPONENT_BIAS - 1; + int32_t biased_exponent = exp2 + NUMBITS + FPBits::EXPONENT_BIAS - 1; // Handle numbers that're too large and get squashed to inf if (biased_exponent >= INF_EXP) { // This indicates an overflow, so we make the result INF and set errno. - output.num = {0, (1 << fputil::FloatProperties::EXPONENT_WIDTH) - 1}; + output.num = {0, (1 << FloatProp::EXPONENT_WIDTH) - 1}; output.error = ERANGE; return output; } - uint32_t amount_to_shift_right = - NUMBITS - fputil::FloatProperties::MANTISSA_WIDTH - 1; + uint32_t amount_to_shift_right = NUMBITS - FloatProp::MANTISSA_WIDTH - 1; // Handle subnormals. if (biased_exponent <= 0) { @@ -814,19 +770,19 @@ LIBC_INLINE FloatConvertReturn binary_exp_to_float(ExpandedFloat init_num, } } - BitsType round_bit_mask = BitsType(1) << (amount_to_shift_right - 1); - BitsType sticky_mask = round_bit_mask - 1; + UIntType round_bit_mask = UIntType(1) << (amount_to_shift_right - 1); + UIntType sticky_mask = round_bit_mask - 1; bool round_bit = static_cast(mantissa & round_bit_mask); bool sticky_bit = static_cast(mantissa & sticky_mask) || truncated; if (amount_to_shift_right < NUMBITS) { // Shift the mantissa and clear the implicit bit. mantissa >>= amount_to_shift_right; - mantissa &= fputil::FloatProperties::MANTISSA_MASK; + mantissa &= FloatProp::MANTISSA_MASK; } else { mantissa = 0; } - bool least_significant_bit = static_cast(mantissa & BitsType(1)); + bool least_significant_bit = static_cast(mantissa & UIntType(1)); // TODO: check that this rounding behavior is correct. @@ -845,7 +801,7 @@ LIBC_INLINE FloatConvertReturn binary_exp_to_float(ExpandedFloat init_num, } } - if (mantissa > fputil::FloatProperties::MANTISSA_MASK) { + if (mantissa > FloatProp::MANTISSA_MASK) { // Rounding causes the exponent to increase. ++biased_exponent; @@ -858,8 +814,7 @@ LIBC_INLINE FloatConvertReturn binary_exp_to_float(ExpandedFloat init_num, output.error = ERANGE; } - output.num = {mantissa & fputil::FloatProperties::MANTISSA_MASK, - biased_exponent}; + output.num = {mantissa & FloatProp::MANTISSA_MASK, biased_exponent}; return output; } @@ -887,14 +842,16 @@ template LIBC_INLINE StrToNumResult> decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, RoundDirection round) { - using BitsType = typename fputil::FPBits::UIntType; + using FPBits = typename fputil::FPBits; + using UIntType = typename FPBits::UIntType; + constexpr uint32_t BASE = 10; constexpr char EXPONENT_MARKER = 'e'; bool truncated = false; bool seen_digit = false; bool after_decimal = false; - BitsType mantissa = 0; + UIntType mantissa = 0; int32_t exponent = 0; size_t index = 0; @@ -905,8 +862,8 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, // the format mantissa * (base ^ exponent) // The loop fills the mantissa with as many digits as it can hold - const BitsType bitstype_max_div_by_base = - cpp::numeric_limits::max() / BASE; + const UIntType bitstype_max_div_by_base = + cpp::numeric_limits::max() / BASE; while (true) { if (isdigit(src[index])) { uint32_t digit = src[index] - '0'; @@ -962,10 +919,10 @@ decimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, // If the result is in the valid range, then we use it. The valid range is // also within the int32 range, so this prevents overflow issues. - if (temp_exponent > fputil::FPBits::MAX_EXPONENT) { - exponent = fputil::FPBits::MAX_EXPONENT; - } else if (temp_exponent < -fputil::FPBits::MAX_EXPONENT) { - exponent = -fputil::FPBits::MAX_EXPONENT; + if (temp_exponent > FPBits::MAX_EXPONENT) { + exponent = FPBits::MAX_EXPONENT; + } else if (temp_exponent < -FPBits::MAX_EXPONENT) { + exponent = -FPBits::MAX_EXPONENT; } else { exponent = static_cast(temp_exponent); } @@ -994,14 +951,16 @@ template LIBC_INLINE StrToNumResult> hexadecimal_string_to_float(const char *__restrict src, const char DECIMAL_POINT, RoundDirection round) { - using BitsType = typename fputil::FPBits::UIntType; + using FPBits = typename fputil::FPBits; + using UIntType = typename FPBits::UIntType; + constexpr uint32_t BASE = 16; constexpr char EXPONENT_MARKER = 'p'; bool truncated = false; bool seen_digit = false; bool after_decimal = false; - BitsType mantissa = 0; + UIntType mantissa = 0; int32_t exponent = 0; size_t index = 0; @@ -1012,8 +971,8 @@ hexadecimal_string_to_float(const char *__restrict src, // the format mantissa * (base ^ exponent) // The loop fills the mantissa with as many digits as it can hold - const BitsType bitstype_max_div_by_base = - cpp::numeric_limits::max() / BASE; + const UIntType bitstype_max_div_by_base = + cpp::numeric_limits::max() / BASE; while (true) { if (isalnum(src[index])) { uint32_t digit = b36_char_to_int(src[index]); @@ -1074,10 +1033,10 @@ hexadecimal_string_to_float(const char *__restrict src, // If the result is in the valid range, then we use it. The valid range is // also within the int32 range, so this prevents overflow issues. - if (temp_exponent > fputil::FPBits::MAX_EXPONENT) { - exponent = fputil::FPBits::MAX_EXPONENT; - } else if (temp_exponent < -fputil::FPBits::MAX_EXPONENT) { - exponent = -fputil::FPBits::MAX_EXPONENT; + if (temp_exponent > FPBits::MAX_EXPONENT) { + exponent = FPBits::MAX_EXPONENT; + } else if (temp_exponent < -FPBits::MAX_EXPONENT) { + exponent = -FPBits::MAX_EXPONENT; } else { exponent = static_cast(temp_exponent); } @@ -1099,8 +1058,10 @@ hexadecimal_string_to_float(const char *__restrict src, // is used as the backend for all of the string to float functions. template LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { - using BitsType = typename fputil::FPBits::UIntType; - fputil::FPBits result = fputil::FPBits(); + using FPBits = typename fputil::FPBits; + using UIntType = typename FPBits::UIntType; + + FPBits result = FPBits(); bool seen_digit = false; char sign = '+'; @@ -1164,7 +1125,7 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { } seen_digit = parse_result.parsed_len != 0; result.set_mantissa(parse_result.value.mantissa); - result.set_unbiased_exponent(parse_result.value.exponent); + result.set_biased_exponent(parse_result.value.exponent); index += parse_result.parsed_len; error = parse_result.error; } else if (tolower(src[index]) == 'n') { // NaN @@ -1172,7 +1133,7 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { tolower(src[index + 2]) == nan_string[2]) { seen_digit = true; index += 3; - BitsType nan_mantissa = 0; + UIntType nan_mantissa = 0; // this handles the case of `NaN(n-character-sequence)`, where the // n-character-sequence is made of 0 or more letters and numbers in any // order. @@ -1186,7 +1147,7 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { if (src[index] == ')') { ++index; if (isdigit(src[left_paren + 1])) { - // This is to prevent errors when BitsType is larger than 64 bits, + // This is to prevent errors when UIntType is larger than 64 bits, // since strtointeger only supports up to 64 bits. This is actually // more than is required by the specification, which says for the // input type "NAN(n-char-sequence)" that "the meaning of @@ -1197,7 +1158,7 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { if (strtoint_result.has_error()) { error = strtoint_result.error; } - nan_mantissa = static_cast(strtoint_result.value); + nan_mantissa = static_cast(strtoint_result.value); if (src[left_paren + 1 + strtoint_result.parsed_len] != ')') nan_mantissa = 0; } @@ -1207,11 +1168,11 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { } nan_mantissa |= fputil::FloatProperties::QUIET_NAN_MASK; if (result.get_sign()) { - result = fputil::FPBits(result.build_quiet_nan(nan_mantissa)); + result = FPBits(result.build_quiet_nan(nan_mantissa)); result.set_sign(true); } else { result.set_sign(false); - result = fputil::FPBits(result.build_quiet_nan(nan_mantissa)); + result = FPBits(result.build_quiet_nan(nan_mantissa)); } } } else if (tolower(src[index]) == 'i') { // INF @@ -1219,9 +1180,9 @@ LIBC_INLINE StrToNumResult strtofloatingpoint(const char *__restrict src) { tolower(src[index + 2]) == inf_string[2]) { seen_digit = true; if (result.get_sign()) - result = fputil::FPBits(result.neg_inf()); + result = FPBits(result.neg_inf()); else - result = fputil::FPBits(result.inf()); + result = FPBits(result.inf()); if (tolower(src[index + 3]) == inf_string[3] && tolower(src[index + 4]) == inf_string[4] && tolower(src[index + 5]) == inf_string[5] && diff --git a/libc/src/math/generic/expf.cpp b/libc/src/math/generic/expf.cpp index 5a938a7a1c22..12f62960fc10 100644 --- a/libc/src/math/generic/expf.cpp +++ b/libc/src/math/generic/expf.cpp @@ -37,7 +37,7 @@ LLVM_LIBC_FUNCTION(float, expf, (float x)) { // When |x| >= 89, |x| < 2^-25, or x is nan if (LIBC_UNLIKELY(x_abs >= 0x42b2'0000U || x_abs <= 0x3280'0000U)) { // |x| < 2^-25 - if (xbits.get_unbiased_exponent() <= 101) { + if (xbits.get_biased_exponent() <= 101) { return 1.0f + x; } diff --git a/libc/src/math/generic/explogxf.h b/libc/src/math/generic/explogxf.h index 512785be2cb8..77ec9cb94e08 100644 --- a/libc/src/math/generic/explogxf.h +++ b/libc/src/math/generic/explogxf.h @@ -285,7 +285,7 @@ LIBC_INLINE static double log2_eval(double x) { (LOG_P1_SIZE - 1); bs.bits &= FPB::FloatProp::MANTISSA_MASK >> LOG_P1_BITS; - bs.set_unbiased_exponent(FPB::FloatProp::EXPONENT_BIAS); + bs.set_biased_exponent(FPB::FloatProp::EXPONENT_BIAS); double dx = (bs.get_val() - 1.0) * LOG_P1_1_OVER[p1]; // Taylor series for log(2,1+x) @@ -311,11 +311,12 @@ LIBC_INLINE static double log_eval(double x) { // p1 is the leading 7 bits of mx, i.e. // p1 * 2^(-7) <= m_x < (p1 + 1) * 2^(-7). - int p1 = (bs.get_mantissa() >> (FPB::FloatProp::MANTISSA_WIDTH - 7)); + int p1 = static_cast(bs.get_mantissa() >> + (FPB::FloatProp::MANTISSA_WIDTH - 7)); // Set bs to (1 + (mx - p1*2^(-7)) bs.bits &= FPB::FloatProp::MANTISSA_MASK >> 7; - bs.set_unbiased_exponent(FPB::FloatProp::EXPONENT_BIAS); + bs.set_biased_exponent(FPB::FloatProp::EXPONENT_BIAS); // dx = (mx - p1*2^(-7)) / (1 + p1*2^(-7)). double dx = (bs.get_val() - 1.0) * ONE_OVER_F[p1]; diff --git a/libc/src/math/generic/hypotf.cpp b/libc/src/math/generic/hypotf.cpp index 3fedeed4ed26..389de3c45029 100644 --- a/libc/src/math/generic/hypotf.cpp +++ b/libc/src/math/generic/hypotf.cpp @@ -19,8 +19,8 @@ LLVM_LIBC_FUNCTION(float, hypotf, (float x, float y)) { FPBits x_bits(x), y_bits(y); - uint16_t x_exp = x_bits.get_unbiased_exponent(); - uint16_t y_exp = y_bits.get_unbiased_exponent(); + uint16_t x_exp = x_bits.get_biased_exponent(); + uint16_t y_exp = y_bits.get_biased_exponent(); uint16_t exp_diff = (x_exp > y_exp) ? (x_exp - y_exp) : (y_exp - x_exp); if (exp_diff >= fputil::MantissaWidth::VALUE + 2) { diff --git a/libc/src/math/generic/log.cpp b/libc/src/math/generic/log.cpp index 46b64df68908..dfa41ad64578 100644 --- a/libc/src/math/generic/log.cpp +++ b/libc/src/math/generic/log.cpp @@ -769,7 +769,7 @@ LLVM_LIBC_FUNCTION(double, log, (double x)) { // Range reduction for log(x_m): // For each x_m, we would like to find r such that: // -2^-8 <= r * x_m - 1 < 2^-7 - int shifted = x_u >> 45; + int shifted = static_cast(x_u >> 45); int index = shifted & 0x7F; double r = RD[index]; diff --git a/libc/src/math/generic/log10.cpp b/libc/src/math/generic/log10.cpp index 38789acc441e..2a801c6e9842 100644 --- a/libc/src/math/generic/log10.cpp +++ b/libc/src/math/generic/log10.cpp @@ -770,7 +770,7 @@ LLVM_LIBC_FUNCTION(double, log10, (double x)) { // Range reduction for log10(x_m): // For each x_m, we would like to find r such that: // -2^-8 <= r * x_m - 1 < 2^-7 - int shifted = x_u >> 45; + int shifted = static_cast(x_u >> 45); int index = shifted & 0x7F; double r = RD[index]; diff --git a/libc/src/math/generic/log10f.cpp b/libc/src/math/generic/log10f.cpp index bc9af75169b1..927af35c98f2 100644 --- a/libc/src/math/generic/log10f.cpp +++ b/libc/src/math/generic/log10f.cpp @@ -187,7 +187,7 @@ LLVM_LIBC_FUNCTION(float, log10f, (float x)) { // Extract 7 leading fractional bits of the mantissa int index = (x_u >> 16) & 0x7F; // Set bits to 1.m - xbits.set_unbiased_exponent(0x7F); + xbits.set_biased_exponent(0x7F); float u = static_cast(xbits); double v; diff --git a/libc/src/math/generic/log1p.cpp b/libc/src/math/generic/log1p.cpp index c6ee8d8f9bbf..c8b45fd57b42 100644 --- a/libc/src/math/generic/log1p.cpp +++ b/libc/src/math/generic/log1p.cpp @@ -880,7 +880,7 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) { fputil::DoubleDouble x_dd{0.0, 0.0}; - uint16_t x_exp = xbits.get_unbiased_exponent(); + uint16_t x_exp = xbits.get_biased_exponent(); if (x_exp >= EXPONENT_BIAS) { // |x| >= 1 @@ -909,7 +909,7 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) { } } else { // |x| < 1 - if (LIBC_UNLIKELY(xbits.get_unbiased_exponent() < + if (LIBC_UNLIKELY(xbits.get_biased_exponent() < EXPONENT_BIAS - MANTISSA_WIDTH - 1)) { // Quick return when |x| < 2^-53. // Since log(1 + x) = x - x^2/2 + x^3/3 - ..., @@ -949,8 +949,9 @@ LLVM_LIBC_FUNCTION(double, log1p, (double x)) { x_u = xhi_bits.uintval(); // Range reduction: // Find k such that |x_hi - k * 2^-7| <= 2^-8. - int idx = ((x_u & MANTISSA_MASK) + (1ULL << (MANTISSA_WIDTH - 8))) >> - (MANTISSA_WIDTH - 7); + int idx = static_cast( + ((x_u & MANTISSA_MASK) + (1ULL << (MANTISSA_WIDTH - 8))) >> + (MANTISSA_WIDTH - 7)); int x_e = xhi_bits.get_exponent() + (idx >> 7); double e_x = static_cast(x_e); diff --git a/libc/src/math/generic/log1pf.cpp b/libc/src/math/generic/log1pf.cpp index 5b4e7edcceb4..fd3cf4647fd0 100644 --- a/libc/src/math/generic/log1pf.cpp +++ b/libc/src/math/generic/log1pf.cpp @@ -56,11 +56,11 @@ LIBC_INLINE float log(double x) { // Get the 8 highest bits, use 7 bits (excluding the implicit hidden bit) for // lookup tables. - int f_index = - xbits.get_mantissa() >> 45; // fputil::MantissaWidth::VALUE - 7 + int f_index = static_cast( + xbits.get_mantissa() >> 45); // fputil::MantissaWidth::VALUE - 7 // Set bits to 1.m - xbits.set_unbiased_exponent(0x3FF); + xbits.set_biased_exponent(0x3FF); FPBits f = xbits; // Clear the lowest 45 bits. diff --git a/libc/src/math/generic/log2.cpp b/libc/src/math/generic/log2.cpp index d72b0931c14b..2ceddf87dfd5 100644 --- a/libc/src/math/generic/log2.cpp +++ b/libc/src/math/generic/log2.cpp @@ -890,7 +890,7 @@ LLVM_LIBC_FUNCTION(double, log2, (double x)) { // Range reduction for log2(x_m): // For each x_m, we would like to find r such that: // -2^-8 <= r * x_m - 1 < 2^-7 - int shifted = x_u >> 45; + int shifted = static_cast(x_u >> 45); int index = shifted & 0x7F; double r = RD[index]; diff --git a/libc/src/math/generic/log2f.cpp b/libc/src/math/generic/log2f.cpp index 7665a90f0923..4bbc9f51c3d2 100644 --- a/libc/src/math/generic/log2f.cpp +++ b/libc/src/math/generic/log2f.cpp @@ -87,10 +87,10 @@ LLVM_LIBC_FUNCTION(float, log2f, (float x)) { m -= 23; } - m += xbits.get_unbiased_exponent(); + m += xbits.get_biased_exponent(); int index = xbits.get_mantissa() >> 16; // Set bits to 1.m - xbits.set_unbiased_exponent(0x7F); + xbits.set_biased_exponent(0x7F); float u = static_cast(xbits); double v; diff --git a/libc/src/math/generic/logf.cpp b/libc/src/math/generic/logf.cpp index 1f689f25931d..2dfada38b972 100644 --- a/libc/src/math/generic/logf.cpp +++ b/libc/src/math/generic/logf.cpp @@ -135,7 +135,7 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) { // rounding mode. if (LIBC_UNLIKELY((x_u & 0x007f'ffffU) == 0)) return static_cast( - static_cast(m + xbits.get_unbiased_exponent()) * LOG_2); + static_cast(m + xbits.get_biased_exponent()) * LOG_2); #endif // LIBC_TARGET_CPU_HAS_FMA uint32_t mant = xbits.get_mantissa(); @@ -146,7 +146,7 @@ LLVM_LIBC_FUNCTION(float, logf, (float x)) { m += static_cast((x_u + (1 << 16)) >> 23); // Set bits to 1.m - xbits.set_unbiased_exponent(0x7F); + xbits.set_biased_exponent(0x7F); float u = static_cast(xbits); double v; diff --git a/libc/src/math/generic/powf.cpp b/libc/src/math/generic/powf.cpp index 5f2e95b44e52..17ccb41fc462 100644 --- a/libc/src/math/generic/powf.cpp +++ b/libc/src/math/generic/powf.cpp @@ -410,8 +410,8 @@ LIBC_INLINE bool is_integer(float x) { LIBC_INLINE bool larger_exponent(double a, double b) { using DoubleBits = typename fputil::FPBits; - return DoubleBits(a).get_unbiased_exponent() >= - DoubleBits(b).get_unbiased_exponent(); + return DoubleBits(a).get_biased_exponent() >= + DoubleBits(b).get_biased_exponent(); } // Calculate 2^(y * log2(x)) in double-double precision. diff --git a/libc/test/src/__support/FPUtil/fpbits_test.cpp b/libc/test/src/__support/FPUtil/fpbits_test.cpp index 52635cc2af09..fa743855c486 100644 --- a/libc/test/src/__support/FPUtil/fpbits_test.cpp +++ b/libc/test/src/__support/FPUtil/fpbits_test.cpp @@ -24,7 +24,7 @@ TEST(LlvmLibcFPBitsTest, FloatType) { FloatBits zero(0.0f); EXPECT_EQ(zero.get_sign(), false); - EXPECT_EQ(zero.get_unbiased_exponent(), static_cast(0)); + EXPECT_EQ(zero.get_biased_exponent(), static_cast(0)); EXPECT_EQ(zero.get_mantissa(), static_cast(0)); EXPECT_EQ(zero.uintval(), static_cast(0x00000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(zero).c_str(), @@ -32,7 +32,7 @@ TEST(LlvmLibcFPBitsTest, FloatType) { FloatBits negzero(-0.0f); EXPECT_EQ(negzero.get_sign(), true); - EXPECT_EQ(negzero.get_unbiased_exponent(), static_cast(0)); + EXPECT_EQ(negzero.get_biased_exponent(), static_cast(0)); EXPECT_EQ(negzero.get_mantissa(), static_cast(0)); EXPECT_EQ(negzero.uintval(), static_cast(0x80000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negzero).c_str(), @@ -40,7 +40,7 @@ TEST(LlvmLibcFPBitsTest, FloatType) { FloatBits one(1.0f); EXPECT_EQ(one.get_sign(), false); - EXPECT_EQ(one.get_unbiased_exponent(), static_cast(0x7F)); + EXPECT_EQ(one.get_biased_exponent(), static_cast(0x7F)); EXPECT_EQ(one.get_mantissa(), static_cast(0)); EXPECT_EQ(one.uintval(), static_cast(0x3F800000)); EXPECT_STREQ(LIBC_NAMESPACE::str(one).c_str(), @@ -48,7 +48,7 @@ TEST(LlvmLibcFPBitsTest, FloatType) { FloatBits negone(-1.0f); EXPECT_EQ(negone.get_sign(), true); - EXPECT_EQ(negone.get_unbiased_exponent(), static_cast(0x7F)); + EXPECT_EQ(negone.get_biased_exponent(), static_cast(0x7F)); EXPECT_EQ(negone.get_mantissa(), static_cast(0)); EXPECT_EQ(negone.uintval(), static_cast(0xBF800000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negone).c_str(), @@ -56,7 +56,7 @@ TEST(LlvmLibcFPBitsTest, FloatType) { FloatBits num(1.125f); EXPECT_EQ(num.get_sign(), false); - EXPECT_EQ(num.get_unbiased_exponent(), static_cast(0x7F)); + EXPECT_EQ(num.get_biased_exponent(), static_cast(0x7F)); EXPECT_EQ(num.get_mantissa(), static_cast(0x00100000)); EXPECT_EQ(num.uintval(), static_cast(0x3F900000)); EXPECT_STREQ(LIBC_NAMESPACE::str(num).c_str(), @@ -64,7 +64,7 @@ TEST(LlvmLibcFPBitsTest, FloatType) { FloatBits negnum(-1.125f); EXPECT_EQ(negnum.get_sign(), true); - EXPECT_EQ(negnum.get_unbiased_exponent(), static_cast(0x7F)); + EXPECT_EQ(negnum.get_biased_exponent(), static_cast(0x7F)); EXPECT_EQ(negnum.get_mantissa(), static_cast(0x00100000)); EXPECT_EQ(negnum.uintval(), static_cast(0xBF900000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negnum).c_str(), @@ -84,7 +84,7 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { DoubleBits zero(0.0); EXPECT_EQ(zero.get_sign(), false); - EXPECT_EQ(zero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(zero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(zero.get_mantissa(), static_cast(0x0000000000000000)); EXPECT_EQ(zero.uintval(), static_cast(0x0000000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(zero).c_str(), @@ -92,7 +92,7 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { DoubleBits negzero(-0.0); EXPECT_EQ(negzero.get_sign(), true); - EXPECT_EQ(negzero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(negzero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(negzero.get_mantissa(), static_cast(0x0000000000000000)); EXPECT_EQ(negzero.uintval(), static_cast(0x8000000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negzero).c_str(), @@ -100,7 +100,7 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { DoubleBits one(1.0); EXPECT_EQ(one.get_sign(), false); - EXPECT_EQ(one.get_unbiased_exponent(), static_cast(0x03FF)); + EXPECT_EQ(one.get_biased_exponent(), static_cast(0x03FF)); EXPECT_EQ(one.get_mantissa(), static_cast(0x0000000000000000)); EXPECT_EQ(one.uintval(), static_cast(0x3FF0000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(one).c_str(), @@ -108,7 +108,7 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { DoubleBits negone(-1.0); EXPECT_EQ(negone.get_sign(), true); - EXPECT_EQ(negone.get_unbiased_exponent(), static_cast(0x03FF)); + EXPECT_EQ(negone.get_biased_exponent(), static_cast(0x03FF)); EXPECT_EQ(negone.get_mantissa(), static_cast(0x0000000000000000)); EXPECT_EQ(negone.uintval(), static_cast(0xBFF0000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negone).c_str(), @@ -116,7 +116,7 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { DoubleBits num(1.125); EXPECT_EQ(num.get_sign(), false); - EXPECT_EQ(num.get_unbiased_exponent(), static_cast(0x03FF)); + EXPECT_EQ(num.get_biased_exponent(), static_cast(0x03FF)); EXPECT_EQ(num.get_mantissa(), static_cast(0x0002000000000000)); EXPECT_EQ(num.uintval(), static_cast(0x3FF2000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(num).c_str(), @@ -124,7 +124,7 @@ TEST(LlvmLibcFPBitsTest, DoubleType) { DoubleBits negnum(-1.125); EXPECT_EQ(negnum.get_sign(), true); - EXPECT_EQ(negnum.get_unbiased_exponent(), static_cast(0x03FF)); + EXPECT_EQ(negnum.get_biased_exponent(), static_cast(0x03FF)); EXPECT_EQ(negnum.get_mantissa(), static_cast(0x0002000000000000)); EXPECT_EQ(negnum.uintval(), static_cast(0xBFF2000000000000)); EXPECT_STREQ(LIBC_NAMESPACE::str(negnum).c_str(), @@ -150,7 +150,7 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LongDoubleBits zero(0.0l); EXPECT_EQ(zero.get_sign(), false); - EXPECT_EQ(zero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(zero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(zero.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(zero.uintval(), static_cast(0x0000000000000000) << 64); @@ -161,7 +161,7 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LongDoubleBits negzero(-0.0l); EXPECT_EQ(negzero.get_sign(), true); - EXPECT_EQ(negzero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(negzero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(negzero.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(negzero.uintval(), static_cast(0x1) << 79); @@ -172,7 +172,7 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LongDoubleBits one(1.0l); EXPECT_EQ(one.get_sign(), false); - EXPECT_EQ(one.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(one.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(one.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(one.uintval(), static_cast(0x3FFF8) << 60); EXPECT_STREQ( @@ -182,7 +182,7 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LongDoubleBits negone(-1.0l); EXPECT_EQ(negone.get_sign(), true); - EXPECT_EQ(negone.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(negone.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(negone.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(negone.uintval(), static_cast(0xBFFF8) << 60); @@ -193,7 +193,7 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LongDoubleBits num(1.125l); EXPECT_EQ(num.get_sign(), false); - EXPECT_EQ(num.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(num.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(num.get_mantissa(), static_cast(0x1) << 60); EXPECT_EQ(num.uintval(), static_cast(0x3FFF9) << 60); EXPECT_STREQ( @@ -203,7 +203,7 @@ TEST(LlvmLibcFPBitsTest, X86LongDoubleType) { LongDoubleBits negnum(-1.125l); EXPECT_EQ(negnum.get_sign(), true); - EXPECT_EQ(negnum.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(negnum.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(negnum.get_mantissa(), static_cast(0x1) << 60); EXPECT_EQ(negnum.uintval(), static_cast(0xBFFF9) << 60); EXPECT_STREQ( @@ -230,7 +230,7 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { LongDoubleBits zero(0.0l); EXPECT_EQ(zero.get_sign(), false); - EXPECT_EQ(zero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(zero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(zero.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(zero.uintval(), static_cast(0x0000000000000000) << 64); @@ -240,7 +240,7 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { LongDoubleBits negzero(-0.0l); EXPECT_EQ(negzero.get_sign(), true); - EXPECT_EQ(negzero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(negzero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(negzero.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(negzero.uintval(), static_cast(0x1) << 127); @@ -250,7 +250,7 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { LongDoubleBits one(1.0l); EXPECT_EQ(one.get_sign(), false); - EXPECT_EQ(one.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(one.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(one.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(one.uintval(), static_cast(0x3FFF) << 112); EXPECT_STREQ(LIBC_NAMESPACE::str(one).c_str(), @@ -259,7 +259,7 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { LongDoubleBits negone(-1.0l); EXPECT_EQ(negone.get_sign(), true); - EXPECT_EQ(negone.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(negone.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(negone.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(negone.uintval(), static_cast(0xBFFF) << 112); @@ -269,7 +269,7 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { LongDoubleBits num(1.125l); EXPECT_EQ(num.get_sign(), false); - EXPECT_EQ(num.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(num.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(num.get_mantissa(), static_cast(0x2) << 108); EXPECT_EQ(num.uintval(), static_cast(0x3FFF2) << 108); EXPECT_STREQ(LIBC_NAMESPACE::str(num).c_str(), @@ -278,7 +278,7 @@ TEST(LlvmLibcFPBitsTest, LongDoubleType) { LongDoubleBits negnum(-1.125l); EXPECT_EQ(negnum.get_sign(), true); - EXPECT_EQ(negnum.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(negnum.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(negnum.get_mantissa(), static_cast(0x2) << 108); EXPECT_EQ(negnum.uintval(), static_cast(0xBFFF2) << 108); EXPECT_STREQ(LIBC_NAMESPACE::str(negnum).c_str(), @@ -303,7 +303,7 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { Float128Bits zero(Float128Bits::zero()); EXPECT_EQ(zero.get_sign(), false); - EXPECT_EQ(zero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(zero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(zero.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(zero.uintval(), static_cast(0x0000000000000000) << 64); @@ -313,7 +313,7 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { Float128Bits negzero(Float128Bits::neg_zero()); EXPECT_EQ(negzero.get_sign(), true); - EXPECT_EQ(negzero.get_unbiased_exponent(), static_cast(0x0000)); + EXPECT_EQ(negzero.get_biased_exponent(), static_cast(0x0000)); EXPECT_EQ(negzero.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(negzero.uintval(), static_cast(0x1) << 127); @@ -323,7 +323,7 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { Float128Bits one(float128(1.0)); EXPECT_EQ(one.get_sign(), false); - EXPECT_EQ(one.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(one.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(one.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(one.uintval(), static_cast(0x3FFF) << 112); EXPECT_STREQ(LIBC_NAMESPACE::str(one).c_str(), @@ -332,7 +332,7 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { Float128Bits negone(float128(-1.0)); EXPECT_EQ(negone.get_sign(), true); - EXPECT_EQ(negone.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(negone.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(negone.get_mantissa(), static_cast(0x0000000000000000) << 64); EXPECT_EQ(negone.uintval(), static_cast(0xBFFF) << 112); @@ -342,7 +342,7 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { Float128Bits num(float128(1.125)); EXPECT_EQ(num.get_sign(), false); - EXPECT_EQ(num.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(num.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(num.get_mantissa(), static_cast(0x2) << 108); EXPECT_EQ(num.uintval(), static_cast(0x3FFF2) << 108); EXPECT_STREQ(LIBC_NAMESPACE::str(num).c_str(), @@ -351,7 +351,7 @@ TEST(LlvmLibcFPBitsTest, Float128Type) { Float128Bits negnum(float128(-1.125)); EXPECT_EQ(negnum.get_sign(), true); - EXPECT_EQ(negnum.get_unbiased_exponent(), static_cast(0x3FFF)); + EXPECT_EQ(negnum.get_biased_exponent(), static_cast(0x3FFF)); EXPECT_EQ(negnum.get_mantissa(), static_cast(0x2) << 108); EXPECT_EQ(negnum.uintval(), static_cast(0xBFFF2) << 108); EXPECT_STREQ(LIBC_NAMESPACE::str(negnum).c_str(), diff --git a/libc/test/src/__support/str_to_float_test.cpp b/libc/test/src/__support/str_to_float_test.cpp index f9d12d95a50b..35f7318fb9c7 100644 --- a/libc/test/src/__support/str_to_float_test.cpp +++ b/libc/test/src/__support/str_to_float_test.cpp @@ -93,45 +93,6 @@ public: } }; -TEST(LlvmLibcStrToFloatTest, LeadingZeroes) { - uint64_t test_num64 = 1; - uint32_t num_of_zeroes = 63; - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(0), 64u); - for (; num_of_zeroes < 64; test_num64 <<= 1, num_of_zeroes--) { - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(test_num64), - num_of_zeroes); - } - - test_num64 = 3; - num_of_zeroes = 62; - for (; num_of_zeroes > 63; test_num64 <<= 1, num_of_zeroes--) { - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(test_num64), - num_of_zeroes); - } - - EXPECT_EQ( - LIBC_NAMESPACE::internal::leading_zeroes(0xffffffffffffffff), - 0u); - - test_num64 = 1; - num_of_zeroes = 63; - for (; num_of_zeroes > 63; - test_num64 = (test_num64 << 1) + 1, num_of_zeroes--) { - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(test_num64), - num_of_zeroes); - } - - uint64_t test_num32 = 1; - num_of_zeroes = 31; - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(0), 32u); - for (; num_of_zeroes < 32; test_num32 <<= 1, num_of_zeroes--) { - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(test_num32), - num_of_zeroes); - } - - EXPECT_EQ(LIBC_NAMESPACE::internal::leading_zeroes(0xffffffff), 0u); -} - TEST_F(LlvmLibcStrToFloatTest, ClingerFastPathFloat64Simple) { clinger_fast_path_test(123, 0, 0xEC00000000000, 1029); clinger_fast_path_test(1234567890123456, 1, 0x5ee2a2eb5a5c0, 1076); diff --git a/libc/test/src/math/LdExpTest.h b/libc/test/src/math/LdExpTest.h index bffa7335fb25..a75c8ef31a2c 100644 --- a/libc/test/src/math/LdExpTest.h +++ b/libc/test/src/math/LdExpTest.h @@ -132,7 +132,7 @@ public: FPBits result_bits(result); ASSERT_FALSE(result_bits.is_zero()); // Verify that the result is indeed subnormal. - ASSERT_EQ(result_bits.get_unbiased_exponent(), uint16_t(0)); + ASSERT_EQ(result_bits.get_biased_exponent(), uint16_t(0)); // But if the exp is so less that normalization leads to zero, then // the result should be zero. result = func(x, -FPBits::MAX_EXPONENT - int(MANTISSA_WIDTH) - 5); diff --git a/libc/test/src/math/NextAfterTest.h b/libc/test/src/math/NextAfterTest.h index 4e450cf5bc1e..57a801dfb28a 100644 --- a/libc/test/src/math/NextAfterTest.h +++ b/libc/test/src/math/NextAfterTest.h @@ -162,30 +162,28 @@ public: result = func(x, 0); FPBits x_bits = FPBits(x); FPBits result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - uint16_t(x_bits.get_unbiased_exponent() - 1)); + ASSERT_EQ(result_bits.get_biased_exponent(), + uint16_t(x_bits.get_biased_exponent() - 1)); ASSERT_EQ(result_bits.get_mantissa(), (UIntType(1) << MantissaWidth::VALUE) - 1); result = func(x, T(33.0)); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - x_bits.get_unbiased_exponent()); + ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), x_bits.get_mantissa() + UIntType(1)); x = -x; result = func(x, 0); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - uint16_t(x_bits.get_unbiased_exponent() - 1)); + ASSERT_EQ(result_bits.get_biased_exponent(), + uint16_t(x_bits.get_biased_exponent() - 1)); ASSERT_EQ(result_bits.get_mantissa(), (UIntType(1) << MantissaWidth::VALUE) - 1); result = func(x, T(-33.0)); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - x_bits.get_unbiased_exponent()); + ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), x_bits.get_mantissa() + UIntType(1)); } }; diff --git a/libc/test/src/math/RoundToIntegerTest.h b/libc/test/src/math/RoundToIntegerTest.h index 6b205869e7a7..1a976e97359e 100644 --- a/libc/test/src/math/RoundToIntegerTest.h +++ b/libc/test/src/math/RoundToIntegerTest.h @@ -126,7 +126,7 @@ public: // We start with 1.0 so that the implicit bit for x86 long doubles // is set. FPBits bits(F(1.0)); - bits.set_unbiased_exponent(EXPONENT_LIMIT + FPBits::EXPONENT_BIAS); + bits.set_biased_exponent(EXPONENT_LIMIT + FPBits::EXPONENT_BIAS); bits.set_sign(1); bits.set_mantissa(0); @@ -190,7 +190,7 @@ public: // We start with 1.0 so that the implicit bit for x86 long doubles // is set. FPBits bits(F(1.0)); - bits.set_unbiased_exponent(EXPONENT_LIMIT + FPBits::EXPONENT_BIAS); + bits.set_biased_exponent(EXPONENT_LIMIT + FPBits::EXPONENT_BIAS); bits.set_sign(1); bits.set_mantissa(UIntType(0x1) << (LIBC_NAMESPACE::fputil::MantissaWidth::VALUE - 1)); diff --git a/libc/test/src/math/smoke/LdExpTest.h b/libc/test/src/math/smoke/LdExpTest.h index bffa7335fb25..a75c8ef31a2c 100644 --- a/libc/test/src/math/smoke/LdExpTest.h +++ b/libc/test/src/math/smoke/LdExpTest.h @@ -132,7 +132,7 @@ public: FPBits result_bits(result); ASSERT_FALSE(result_bits.is_zero()); // Verify that the result is indeed subnormal. - ASSERT_EQ(result_bits.get_unbiased_exponent(), uint16_t(0)); + ASSERT_EQ(result_bits.get_biased_exponent(), uint16_t(0)); // But if the exp is so less that normalization leads to zero, then // the result should be zero. result = func(x, -FPBits::MAX_EXPONENT - int(MANTISSA_WIDTH) - 5); diff --git a/libc/test/src/math/smoke/NextAfterTest.h b/libc/test/src/math/smoke/NextAfterTest.h index 1b082050a598..29098e0f49a4 100644 --- a/libc/test/src/math/smoke/NextAfterTest.h +++ b/libc/test/src/math/smoke/NextAfterTest.h @@ -173,30 +173,28 @@ public: result = func(x, 0); FPBits x_bits = FPBits(x); FPBits result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - uint16_t(x_bits.get_unbiased_exponent() - 1)); + ASSERT_EQ(result_bits.get_biased_exponent(), + uint16_t(x_bits.get_biased_exponent() - 1)); ASSERT_EQ(result_bits.get_mantissa(), (UIntType(1) << MantissaWidth::VALUE) - 1); result = func(x, T(33.0)); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - x_bits.get_unbiased_exponent()); + ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), x_bits.get_mantissa() + UIntType(1)); x = -x; result = func(x, 0); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - uint16_t(x_bits.get_unbiased_exponent() - 1)); + ASSERT_EQ(result_bits.get_biased_exponent(), + uint16_t(x_bits.get_biased_exponent() - 1)); ASSERT_EQ(result_bits.get_mantissa(), (UIntType(1) << MantissaWidth::VALUE) - 1); result = func(x, T(-33.0)); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - x_bits.get_unbiased_exponent()); + ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), x_bits.get_mantissa() + UIntType(1)); } }; diff --git a/libc/test/src/math/smoke/NextTowardTest.h b/libc/test/src/math/smoke/NextTowardTest.h index 4d27592a1142..111d8017e691 100644 --- a/libc/test/src/math/smoke/NextTowardTest.h +++ b/libc/test/src/math/smoke/NextTowardTest.h @@ -187,30 +187,28 @@ public: result = func(x, 0); FPBits x_bits = FPBits(x); FPBits result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - uint16_t(x_bits.get_unbiased_exponent() - 1)); + ASSERT_EQ(result_bits.get_biased_exponent(), + uint16_t(x_bits.get_biased_exponent() - 1)); ASSERT_EQ(result_bits.get_mantissa(), (UIntType(1) << MantissaWidth::VALUE) - 1); result = func(x, 33.0); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - x_bits.get_unbiased_exponent()); + ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), x_bits.get_mantissa() + UIntType(1)); x = -x; result = func(x, 0); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - uint16_t(x_bits.get_unbiased_exponent() - 1)); + ASSERT_EQ(result_bits.get_biased_exponent(), + uint16_t(x_bits.get_biased_exponent() - 1)); ASSERT_EQ(result_bits.get_mantissa(), (UIntType(1) << MantissaWidth::VALUE) - 1); result = func(x, -33.0); result_bits = FPBits(result); - ASSERT_EQ(result_bits.get_unbiased_exponent(), - x_bits.get_unbiased_exponent()); + ASSERT_EQ(result_bits.get_biased_exponent(), x_bits.get_biased_exponent()); ASSERT_EQ(result_bits.get_mantissa(), x_bits.get_mantissa() + UIntType(1)); } }; diff --git a/libc/test/utils/FPUtil/x86_long_double_test.cpp b/libc/test/utils/FPUtil/x86_long_double_test.cpp index 6ca9b16377c0..cea43c1a6fa4 100644 --- a/libc/test/utils/FPUtil/x86_long_double_test.cpp +++ b/libc/test/utils/FPUtil/x86_long_double_test.cpp @@ -22,7 +22,7 @@ TEST(LlvmLibcX86LongDoubleTest, is_nan) { constexpr uint32_t COUNT = 100'000; FPBits bits(0.0l); - bits.set_unbiased_exponent(FPBits::MAX_EXPONENT); + bits.set_biased_exponent(FPBits::MAX_EXPONENT); for (unsigned int i = 0; i < COUNT; ++i) { // If exponent has the max value and the implicit bit is 0, // then the number is a NaN for all values of mantissa. @@ -43,7 +43,7 @@ TEST(LlvmLibcX86LongDoubleTest, is_nan) { ASSERT_TRUE(bits.is_nan()); } - bits.set_unbiased_exponent(1); + bits.set_biased_exponent(1); bits.set_implicit_bit(0); for (unsigned int i = 0; i < COUNT; ++i) { // If exponent is non-zero and also not max, and the implicit bit is 0, @@ -54,7 +54,7 @@ TEST(LlvmLibcX86LongDoubleTest, is_nan) { ASSERT_TRUE(bits.is_nan()); } - bits.set_unbiased_exponent(1); + bits.set_biased_exponent(1); bits.set_implicit_bit(1); for (unsigned int i = 0; i < COUNT; ++i) { // If exponent is non-zero and also not max, and the implicit bit is 1, @@ -65,7 +65,7 @@ TEST(LlvmLibcX86LongDoubleTest, is_nan) { ASSERT_FALSE(bits.is_nan()); } - bits.set_unbiased_exponent(0); + bits.set_biased_exponent(0); bits.set_implicit_bit(1); for (unsigned int i = 0; i < COUNT; ++i) { // If exponent is zero, then the number is a valid but denormal value. @@ -75,7 +75,7 @@ TEST(LlvmLibcX86LongDoubleTest, is_nan) { ASSERT_FALSE(bits.is_nan()); } - bits.set_unbiased_exponent(0); + bits.set_biased_exponent(0); bits.set_implicit_bit(0); for (unsigned int i = 0; i < COUNT; ++i) { // If exponent is zero, then the number is a valid but denormal value. diff --git a/libc/utils/MPFRWrapper/MPFRUtils.cpp b/libc/utils/MPFRWrapper/MPFRUtils.cpp index e1213e00ff36..e3dffadf4ed9 100644 --- a/libc/utils/MPFRWrapper/MPFRUtils.cpp +++ b/libc/utils/MPFRWrapper/MPFRUtils.cpp @@ -458,9 +458,9 @@ public: int thisExponent = fputil::FPBits(thisAsT).get_exponent(); int inputExponent = fputil::FPBits(input).get_exponent(); // Adjust the exponents for denormal numbers. - if (fputil::FPBits(thisAsT).get_unbiased_exponent() == 0) + if (fputil::FPBits(thisAsT).get_biased_exponent() == 0) ++thisExponent; - if (fputil::FPBits(input).get_unbiased_exponent() == 0) + if (fputil::FPBits(input).get_biased_exponent() == 0) ++inputExponent; if (thisAsT * input < 0 || thisExponent == inputExponent) { @@ -483,9 +483,9 @@ public: int minExponent = fputil::FPBits(min).get_exponent(); int maxExponent = fputil::FPBits(max).get_exponent(); // Adjust the exponents for denormal numbers. - if (fputil::FPBits(min).get_unbiased_exponent() == 0) + if (fputil::FPBits(min).get_biased_exponent() == 0) ++minExponent; - if (fputil::FPBits(max).get_unbiased_exponent() == 0) + if (fputil::FPBits(max).get_biased_exponent() == 0) ++maxExponent; MPFRNumber minMPFR(min); diff --git a/libcxx/docs/ReleaseNotes/18.rst b/libcxx/docs/ReleaseNotes/18.rst index abefe4c28ca9..9e509db6359c 100644 --- a/libcxx/docs/ReleaseNotes/18.rst +++ b/libcxx/docs/ReleaseNotes/18.rst @@ -50,6 +50,7 @@ Implemented Papers - P0053R7 - C++ Synchronized Buffered Ostream (in the experimental library) - P2467R1 - Support exclusive mode for fstreams - P0020R6 - Floating Point Atomic +- P2905R2 - Runtime format strings - P2918R2 - Runtime format strings II - P2871R3 - Remove Deprecated Unicode Conversion Facets from C++26 - P2870R3 - Remove basic_string::reserve() diff --git a/libcxx/docs/Status/Cxx2cPapers.csv b/libcxx/docs/Status/Cxx2cPapers.csv index 1d071b7ebcb4..ff83648aa768 100644 --- a/libcxx/docs/Status/Cxx2cPapers.csv +++ b/libcxx/docs/Status/Cxx2cPapers.csv @@ -30,7 +30,7 @@ "`P0543R3 `__","LWG","Saturation arithmetic","Kona November 2023","","","" "`P2407R5 `__","LWG","Freestanding Library: Partial Classes","Kona November 2023","","","" "`P2546R5 `__","LWG","Debugging Support","Kona November 2023","","","" -"`P2905R2 `__","LWG","Runtime format strings","Kona November 2023","","","|format| |DR|" +"`P2905R2 `__","LWG","Runtime format strings","Kona November 2023","|Complete|","18.0","|format| |DR|" "`P2918R2 `__","LWG","Runtime format strings II","Kona November 2023","|Complete|","18.0","|format|" "`P2909R4 `__","LWG","Fix formatting of code units as integers (Dude, where’s my ``char``?)","Kona November 2023","|Complete|","18.0","|format| |DR|" "`P0952R2 `__","LWG","A new specification for ``std::generate_canonical``","Kona November 2023","","","" diff --git a/libcxx/docs/Status/FormatIssues.csv b/libcxx/docs/Status/FormatIssues.csv index 005de97405f7..efb3e484f357 100644 --- a/libcxx/docs/Status/FormatIssues.csv +++ b/libcxx/docs/Status/FormatIssues.csv @@ -17,7 +17,7 @@ Number,Name,Standard,Assignee,Status,First released version "`P2510R3 `__","Formatting pointers","C++26","Mark de Wever","|Complete|",17.0 "`P2757R3 `__","Type-checking format args","C++26","","", "`P2637R3 `__","Member ``visit``","C++26","","", -"`P2905R2 `__","Runtime format strings","C++26 DR","Mark de Wever","|In Progress|" +"`P2905R2 `__","Runtime format strings","C++26 DR","Mark de Wever","|Complete|",18.0 "`P2918R2 `__","Runtime format strings II","C++26","Mark de Wever","|Complete|",18.0 "`P2909R4 `__","Fix formatting of code units as integers (Dude, where’s my ``char``?)","C++26 DR","Mark de Wever","|Complete|",18.0 `P1361 `_,"Integration of chrono with text formatting","C++20",Mark de Wever,|In Progress|, diff --git a/libcxx/include/__format/format_arg_store.h b/libcxx/include/__format/format_arg_store.h index 2962962ab5d1..64ee12440b62 100644 --- a/libcxx/include/__format/format_arg_store.h +++ b/libcxx/include/__format/format_arg_store.h @@ -206,8 +206,8 @@ _LIBCPP_HIDE_FROM_ABI basic_format_arg<_Context> __create_format_arg(_Tp& __valu } template -_LIBCPP_HIDE_FROM_ABI void __create_packed_storage(uint64_t& __types, __basic_format_arg_value<_Context>* __values, - _Args&&... __args) noexcept { +_LIBCPP_HIDE_FROM_ABI void +__create_packed_storage(uint64_t& __types, __basic_format_arg_value<_Context>* __values, _Args&... __args) noexcept { int __shift = 0; ( [&] { @@ -224,7 +224,7 @@ _LIBCPP_HIDE_FROM_ABI void __create_packed_storage(uint64_t& __types, __basic_fo } template -_LIBCPP_HIDE_FROM_ABI void __store_basic_format_arg(basic_format_arg<_Context>* __data, _Args&&... __args) noexcept { +_LIBCPP_HIDE_FROM_ABI void __store_basic_format_arg(basic_format_arg<_Context>* __data, _Args&... __args) noexcept { ([&] { *__data++ = __format::__create_format_arg<_Context>(__args); }(), ...); } diff --git a/libcxx/include/__format/format_functions.h b/libcxx/include/__format/format_functions.h index 164592d2ec15..8b2111f0e287 100644 --- a/libcxx/include/__format/format_functions.h +++ b/libcxx/include/__format/format_functions.h @@ -63,15 +63,15 @@ using wformat_args = basic_format_args; #endif template -_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI __format_arg_store<_Context, _Args...> make_format_args(_Args&&... __args) { - return std::__format_arg_store<_Context, _Args...>(__args...); +_LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI __format_arg_store<_Context, _Args...> make_format_args(_Args&... __args) { + return _VSTD::__format_arg_store<_Context, _Args...>(__args...); } # ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS template _LIBCPP_NODISCARD_EXT _LIBCPP_HIDE_FROM_ABI __format_arg_store -make_wformat_args(_Args&&... __args) { - return std::__format_arg_store(__args...); +make_wformat_args(_Args&... __args) { + return _VSTD::__format_arg_store(__args...); } # endif diff --git a/libcxx/include/__locale b/libcxx/include/__locale index ab36939f9c14..6eb892287160 100644 --- a/libcxx/include/__locale +++ b/libcxx/include/__locale @@ -389,7 +389,7 @@ public: static const mask punct = _ISpunct; static const mask xdigit = _ISxdigit; static const mask blank = _ISblank; -#if defined(__mips__) +#if defined(__mips__) || (BYTE_ORDER == BIG_ENDIAN) static const mask __regex_word = static_cast(_ISbit(15)); #else static const mask __regex_word = 0x80; diff --git a/libcxx/include/__mdspan/mdspan.h b/libcxx/include/__mdspan/mdspan.h index 58f3b9cf1b18..684828eb90ec 100644 --- a/libcxx/include/__mdspan/mdspan.h +++ b/libcxx/include/__mdspan/mdspan.h @@ -244,9 +244,14 @@ public: _LIBCPP_HIDE_FROM_ABI constexpr const mapping_type& mapping() const noexcept { return __map_; }; _LIBCPP_HIDE_FROM_ABI constexpr const accessor_type& accessor() const noexcept { return __acc_; }; - _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_unique() { return mapping_type::is_always_unique(); }; - _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_exhaustive() { return mapping_type::is_always_exhaustive(); }; - _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_strided() { return mapping_type::is_always_strided(); }; + // per LWG-4021 "mdspan::is_always_meow() should be noexcept" + _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_unique() noexcept { return mapping_type::is_always_unique(); }; + _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_exhaustive() noexcept { + return mapping_type::is_always_exhaustive(); + }; + _LIBCPP_HIDE_FROM_ABI static constexpr bool is_always_strided() noexcept { + return mapping_type::is_always_strided(); + }; _LIBCPP_HIDE_FROM_ABI constexpr bool is_unique() const { return __map_.is_unique(); }; _LIBCPP_HIDE_FROM_ABI constexpr bool is_exhaustive() const { return __map_.is_exhaustive(); }; diff --git a/libcxx/include/__random/cauchy_distribution.h b/libcxx/include/__random/cauchy_distribution.h index b057074ab8d8..44d7e0e27c14 100644 --- a/libcxx/include/__random/cauchy_distribution.h +++ b/libcxx/include/__random/cauchy_distribution.h @@ -28,6 +28,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS cauchy_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/chi_squared_distribution.h b/libcxx/include/__random/chi_squared_distribution.h index 5cae56b26417..ce17a283d495 100644 --- a/libcxx/include/__random/chi_squared_distribution.h +++ b/libcxx/include/__random/chi_squared_distribution.h @@ -11,6 +11,7 @@ #include <__config> #include <__random/gamma_distribution.h> +#include <__random/is_valid.h> #include #include @@ -26,6 +27,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS chi_squared_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/exponential_distribution.h b/libcxx/include/__random/exponential_distribution.h index e5f0dad8cb88..e9df4de90e74 100644 --- a/libcxx/include/__random/exponential_distribution.h +++ b/libcxx/include/__random/exponential_distribution.h @@ -29,6 +29,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS exponential_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/extreme_value_distribution.h b/libcxx/include/__random/extreme_value_distribution.h index 4afc27598157..f8f18ebf7896 100644 --- a/libcxx/include/__random/extreme_value_distribution.h +++ b/libcxx/include/__random/extreme_value_distribution.h @@ -28,6 +28,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS extreme_value_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/fisher_f_distribution.h b/libcxx/include/__random/fisher_f_distribution.h index 1915cdb06a84..8da689a3b0f4 100644 --- a/libcxx/include/__random/fisher_f_distribution.h +++ b/libcxx/include/__random/fisher_f_distribution.h @@ -27,6 +27,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS fisher_f_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/gamma_distribution.h b/libcxx/include/__random/gamma_distribution.h index e2f04ad51727..bdf8e141ece7 100644 --- a/libcxx/include/__random/gamma_distribution.h +++ b/libcxx/include/__random/gamma_distribution.h @@ -29,6 +29,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS gamma_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/is_valid.h b/libcxx/include/__random/is_valid.h index 113452bc5d35..bde4733099d6 100644 --- a/libcxx/include/__random/is_valid.h +++ b/libcxx/include/__random/is_valid.h @@ -23,6 +23,20 @@ _LIBCPP_BEGIN_NAMESPACE_STD +// [rand.req.genl]/1.4: +// The effect of instantiating a template that has a template type parameter +// named RealType is undefined unless the corresponding template argument is +// cv-unqualified and is one of float, double, or long double. + +template +struct __libcpp_random_is_valid_realtype : false_type {}; +template <> +struct __libcpp_random_is_valid_realtype : true_type {}; +template <> +struct __libcpp_random_is_valid_realtype : true_type {}; +template <> +struct __libcpp_random_is_valid_realtype : true_type {}; + // [rand.req.genl]/1.5: // The effect of instantiating a template that has a template type parameter // named IntType is undefined unless the corresponding template argument is diff --git a/libcxx/include/__random/lognormal_distribution.h b/libcxx/include/__random/lognormal_distribution.h index c30f9adf103a..47db56ee4529 100644 --- a/libcxx/include/__random/lognormal_distribution.h +++ b/libcxx/include/__random/lognormal_distribution.h @@ -10,6 +10,7 @@ #define _LIBCPP___RANDOM_LOGNORMAL_DISTRIBUTION_H #include <__config> +#include <__random/is_valid.h> #include <__random/normal_distribution.h> #include #include @@ -27,6 +28,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS lognormal_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/normal_distribution.h b/libcxx/include/__random/normal_distribution.h index 14649f1757ff..9d68ea6118fd 100644 --- a/libcxx/include/__random/normal_distribution.h +++ b/libcxx/include/__random/normal_distribution.h @@ -28,6 +28,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS normal_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/piecewise_constant_distribution.h b/libcxx/include/__random/piecewise_constant_distribution.h index f5b90290209b..3a60e5533d77 100644 --- a/libcxx/include/__random/piecewise_constant_distribution.h +++ b/libcxx/include/__random/piecewise_constant_distribution.h @@ -29,6 +29,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS piecewise_constant_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/piecewise_linear_distribution.h b/libcxx/include/__random/piecewise_linear_distribution.h index 824e089b0c34..895f0f2dfe94 100644 --- a/libcxx/include/__random/piecewise_linear_distribution.h +++ b/libcxx/include/__random/piecewise_linear_distribution.h @@ -29,6 +29,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS piecewise_linear_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/student_t_distribution.h b/libcxx/include/__random/student_t_distribution.h index d52510dca4ed..0879a411c0c5 100644 --- a/libcxx/include/__random/student_t_distribution.h +++ b/libcxx/include/__random/student_t_distribution.h @@ -29,6 +29,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS student_t_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/uniform_real_distribution.h b/libcxx/include/__random/uniform_real_distribution.h index c5ee527353ab..28b6e9181c5f 100644 --- a/libcxx/include/__random/uniform_real_distribution.h +++ b/libcxx/include/__random/uniform_real_distribution.h @@ -27,6 +27,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS uniform_real_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/__random/weibull_distribution.h b/libcxx/include/__random/weibull_distribution.h index 099cfdd6171b..996799281031 100644 --- a/libcxx/include/__random/weibull_distribution.h +++ b/libcxx/include/__random/weibull_distribution.h @@ -11,6 +11,7 @@ #include <__config> #include <__random/exponential_distribution.h> +#include <__random/is_valid.h> #include #include #include @@ -27,6 +28,9 @@ _LIBCPP_BEGIN_NAMESPACE_STD template class _LIBCPP_TEMPLATE_VIS weibull_distribution { + static_assert(__libcpp_random_is_valid_realtype<_RealType>::value, + "RealType must be a supported floating-point type"); + public: // types typedef _RealType result_type; diff --git a/libcxx/include/any b/libcxx/include/any index 516fd6ddb23c..7dcffc46a60d 100644 --- a/libcxx/include/any +++ b/libcxx/include/any @@ -194,9 +194,7 @@ namespace __any_imp if (__id && *__id == typeid(_Tp)) return true; #endif - if (!__id && __fallback_id == __any_imp::__get_fallback_typeid<_Tp>()) - return true; - return false; + return !__id && __fallback_id == __any_imp::__get_fallback_typeid<_Tp>(); } template diff --git a/libcxx/include/format b/libcxx/include/format index 7b8d5922cb49..ab9b336d0cda 100644 --- a/libcxx/include/format +++ b/libcxx/include/format @@ -177,10 +177,10 @@ namespace std { template format-arg-store - make_format_args(Args&&... args); + make_format_args(Args&... args); template format-arg-store - make_wformat_args(Args&&... args); + make_wformat_args(Args&... args); // [format.error], class format_error class format_error; diff --git a/libcxx/include/mdspan b/libcxx/include/mdspan index d55cdc4a4df5..c13d9eef001a 100644 --- a/libcxx/include/mdspan +++ b/libcxx/include/mdspan @@ -334,11 +334,12 @@ namespace std { constexpr const mapping_type& mapping() const noexcept { return map_; } constexpr const accessor_type& accessor() const noexcept { return acc_; } - static constexpr bool is_always_unique() + // per LWG-4021 "mdspan::is_always_meow() should be noexcept" + static constexpr bool is_always_unique() noexcept { return mapping_type::is_always_unique(); } - static constexpr bool is_always_exhaustive() + static constexpr bool is_always_exhaustive() noexcept { return mapping_type::is_always_exhaustive(); } - static constexpr bool is_always_strided() + static constexpr bool is_always_strided() noexcept { return mapping_type::is_always_strided(); } constexpr bool is_unique() const diff --git a/libcxx/modules/CMakeLists.txt b/libcxx/modules/CMakeLists.txt index 395226fb3728..fae6448a7eec 100644 --- a/libcxx/modules/CMakeLists.txt +++ b/libcxx/modules/CMakeLists.txt @@ -118,6 +118,30 @@ set(LIBCXX_MODULE_STD_SOURCES std/version.inc ) +set(LIBCXX_MODULE_STD_COMPAT_SOURCES + std.compat/cassert.inc + std.compat/cctype.inc + std.compat/cerrno.inc + std.compat/cfenv.inc + std.compat/cfloat.inc + std.compat/cinttypes.inc + std.compat/climits.inc + std.compat/clocale.inc + std.compat/cmath.inc + std.compat/csetjmp.inc + std.compat/csignal.inc + std.compat/cstdarg.inc + std.compat/cstddef.inc + std.compat/cstdint.inc + std.compat/cstdio.inc + std.compat/cstdlib.inc + std.compat/cstring.inc + std.compat/ctime.inc + std.compat/cuchar.inc + std.compat/cwchar.inc + std.compat/cwctype.inc +) + # TODO MODULES the CMakeLists.txt in the install directory is only temporary # When that is removed the configured file can use the substitution # LIBCXX_GENERATED_INCLUDE_TARGET_DIR avoiding this set. @@ -154,10 +178,25 @@ configure_file( @ONLY ) +set(LIBCXX_MODULE_STD_COMPAT_INCLUDE_SOURCES) +foreach(file ${LIBCXX_MODULE_STD_COMPAT_SOURCES}) + set( + LIBCXX_MODULE_STD_COMPAT_INCLUDE_SOURCES + "${LIBCXX_MODULE_STD_COMPAT_INCLUDE_SOURCES}#include \"${file}\"\n" + ) +endforeach() + +configure_file( + "std.compat.cppm.in" + "${LIBCXX_GENERATED_MODULE_DIR}/std.compat.cppm" + @ONLY +) + set(_all_modules) list(APPEND _all_modules "${LIBCXX_GENERATED_MODULE_DIR}/CMakeLists.txt") list(APPEND _all_modules "${LIBCXX_GENERATED_MODULE_DIR}/std.cppm") -foreach(file ${LIBCXX_MODULE_STD_SOURCES}) +list(APPEND _all_modules "${LIBCXX_GENERATED_MODULE_DIR}/std.compat.cppm") +foreach(file ${LIBCXX_MODULE_STD_SOURCES} ${LIBCXX_MODULE_STD_COMPAT_SOURCES}) set(src "${CMAKE_CURRENT_SOURCE_DIR}/${file}") set(dst "${LIBCXX_GENERATED_MODULE_DIR}/${file}") add_custom_command(OUTPUT ${dst} diff --git a/libcxx/modules/CMakeLists.txt.in b/libcxx/modules/CMakeLists.txt.in index dca3b25155a5..b02b68915b8f 100644 --- a/libcxx/modules/CMakeLists.txt.in +++ b/libcxx/modules/CMakeLists.txt.in @@ -29,6 +29,8 @@ macro(compile_define_if condition def) endif() endmacro() +### STD + add_library(std) target_sources(std PUBLIC FILE_SET cxx_modules TYPE CXX_MODULES FILES @@ -52,3 +54,29 @@ set_target_properties(std PROPERTIES OUTPUT_NAME "c++std" ) + +### STD.COMPAT + +add_library(std.compat) +target_sources(std.compat + PUBLIC FILE_SET cxx_modules TYPE CXX_MODULES FILES + std.compat.cppm +) + +target_include_directories(std.compat SYSTEM PRIVATE @LIBCXX_CONFIGURED_INCLUDE_DIRS@) + +if (NOT @LIBCXX_ENABLE_EXCEPTIONS@) + target_compile_options(std.compat PUBLIC -fno-exceptions) +endif() + +target_compile_options(std.compat + PUBLIC + -nostdinc++ + -Wno-reserved-module-identifier + -Wno-reserved-user-defined-literal + @LIBCXX_COMPILE_FLAGS@ +) +set_target_properties(std.compat + PROPERTIES + OUTPUT_NAME "c++std.compat" +) diff --git a/libcxx/modules/std.compat.cppm.in b/libcxx/modules/std.compat.cppm.in new file mode 100644 index 000000000000..f199e194e60b --- /dev/null +++ b/libcxx/modules/std.compat.cppm.in @@ -0,0 +1,208 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// WARNING, this entire header is generated by +// utils/generate_libcxx_cppm_in.py +// DO NOT MODIFY! + +module; + +#include <__config> + +// The headers of Table 24: C++ library headers [tab:headers.cpp] +// and the headers of Table 25: C++ headers for C library facilities [tab:headers.cpp.c] +#include +#include +#include +#if !defined(_LIBCPP_HAS_NO_ATOMIC_HEADER) +# include +#endif +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS) +# include +#endif +#if !defined(_LIBCPP_HAS_NO_WIDE_CHARACTERS) +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#include +#include +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#include +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#include +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#if !defined(_LIBCPP_HAS_NO_LOCALIZATION) +# include +#endif +#include +#if !defined(_LIBCPP_HAS_NO_THREADS) +# include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// *** Headers not yet available *** +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() +#if __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() + +export module std.compat; + +@LIBCXX_MODULE_STD_INCLUDE_SOURCES@ +@LIBCXX_MODULE_STD_COMPAT_INCLUDE_SOURCES@ \ No newline at end of file diff --git a/libcxx/modules/std.compat/cassert.inc b/libcxx/modules/std.compat/cassert.inc new file mode 100644 index 000000000000..ac0533d14e9a --- /dev/null +++ b/libcxx/modules/std.compat/cassert.inc @@ -0,0 +1,12 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // This module exports nothing. +} // export diff --git a/libcxx/modules/std.compat/cctype.inc b/libcxx/modules/std.compat/cctype.inc new file mode 100644 index 000000000000..56fb45a374a5 --- /dev/null +++ b/libcxx/modules/std.compat/cctype.inc @@ -0,0 +1,25 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::isalnum; + using ::isalpha; + using ::isblank; + using ::iscntrl; + using ::isdigit; + using ::isgraph; + using ::islower; + using ::isprint; + using ::ispunct; + using ::isspace; + using ::isupper; + using ::isxdigit; + using ::tolower; + using ::toupper; +} // export diff --git a/libcxx/modules/std.compat/cerrno.inc b/libcxx/modules/std.compat/cerrno.inc new file mode 100644 index 000000000000..ac0533d14e9a --- /dev/null +++ b/libcxx/modules/std.compat/cerrno.inc @@ -0,0 +1,12 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // This module exports nothing. +} // export diff --git a/libcxx/modules/std.compat/cfenv.inc b/libcxx/modules/std.compat/cfenv.inc new file mode 100644 index 000000000000..50128463d6a9 --- /dev/null +++ b/libcxx/modules/std.compat/cfenv.inc @@ -0,0 +1,29 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // types + using ::fenv_t; + using ::fexcept_t; + + // functions + using ::feclearexcept; + using ::fegetexceptflag; + using ::feraiseexcept; + using ::fesetexceptflag; + using ::fetestexcept; + + using ::fegetround; + using ::fesetround; + + using ::fegetenv; + using ::feholdexcept; + using ::fesetenv; + using ::feupdateenv; +} // export diff --git a/libcxx/modules/std.compat/cfloat.inc b/libcxx/modules/std.compat/cfloat.inc new file mode 100644 index 000000000000..ac0533d14e9a --- /dev/null +++ b/libcxx/modules/std.compat/cfloat.inc @@ -0,0 +1,12 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // This module exports nothing. +} // export diff --git a/libcxx/modules/std.compat/cinttypes.inc b/libcxx/modules/std.compat/cinttypes.inc new file mode 100644 index 000000000000..a64c088d0d6f --- /dev/null +++ b/libcxx/modules/std.compat/cinttypes.inc @@ -0,0 +1,25 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::imaxdiv_t; + + using ::imaxabs; + using ::imaxdiv; + using ::strtoimax; + using ::strtoumax; + using ::wcstoimax; + using ::wcstoumax; + + // abs is conditionally here, but always present in cmath.cppm. To avoid + // conflicing declarations omit the using here. + + // div is conditionally here, but always present in cstdlib.cppm. To avoid + // conflicing declarations omit the using here. +} // export diff --git a/libcxx/modules/std.compat/climits.inc b/libcxx/modules/std.compat/climits.inc new file mode 100644 index 000000000000..ac0533d14e9a --- /dev/null +++ b/libcxx/modules/std.compat/climits.inc @@ -0,0 +1,12 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // This module exports nothing. +} // export diff --git a/libcxx/modules/std.compat/clocale.inc b/libcxx/modules/std.compat/clocale.inc new file mode 100644 index 000000000000..d9785a737943 --- /dev/null +++ b/libcxx/modules/std.compat/clocale.inc @@ -0,0 +1,17 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { +#ifndef _LIBCPP_HAS_NO_LOCALIZATION + using ::lconv; + + using ::localeconv; + using ::setlocale; +#endif // _LIBCPP_HAS_NO_LOCALIZATION +} // export diff --git a/libcxx/modules/std.compat/cmath.inc b/libcxx/modules/std.compat/cmath.inc new file mode 100644 index 000000000000..de5379275c5f --- /dev/null +++ b/libcxx/modules/std.compat/cmath.inc @@ -0,0 +1,268 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::double_t; + using ::float_t; + + using ::acos; + using ::acosf; + using ::acosl; + + using ::asin; + using ::asinf; + using ::asinl; + + using ::atan; + using ::atanf; + using ::atanl; + + using ::atan2; + using ::atan2f; + using ::atan2l; + + using ::cos; + using ::cosf; + using ::cosl; + + using ::sin; + using ::sinf; + using ::sinl; + + using ::tan; + using ::tanf; + using ::tanl; + + using ::acosh; + using ::acoshf; + using ::acoshl; + + using ::asinh; + using ::asinhf; + using ::asinhl; + + using ::atanh; + using ::atanhf; + using ::atanhl; + + using ::cosh; + using ::coshf; + using ::coshl; + + using ::sinh; + using ::sinhf; + using ::sinhl; + + using ::tanh; + using ::tanhf; + using ::tanhl; + + using ::exp; + using ::expf; + using ::expl; + + using ::exp2; + using ::exp2f; + using ::exp2l; + + using ::expm1; + using ::expm1f; + using ::expm1l; + + using ::frexp; + using ::frexpf; + using ::frexpl; + + using ::ilogb; + using ::ilogbf; + using ::ilogbl; + + using ::ldexp; + using ::ldexpf; + using ::ldexpl; + + using ::log; + using ::logf; + using ::logl; + + using ::log10; + using ::log10f; + using ::log10l; + + using ::log1p; + using ::log1pf; + using ::log1pl; + + using ::log2; + using ::log2f; + using ::log2l; + + using ::logb; + using ::logbf; + using ::logbl; + + using ::modf; + using ::modff; + using ::modfl; + + using ::scalbn; + using ::scalbnf; + using ::scalbnl; + + using ::scalbln; + using ::scalblnf; + using ::scalblnl; + + using ::cbrt; + using ::cbrtf; + using ::cbrtl; + + // [c.math.abs], absolute values + using ::abs; + + using ::fabs; + using ::fabsf; + using ::fabsl; + + using ::hypot; + using ::hypotf; + using ::hypotl; + + // [c.math.hypot3], three-dimensional hypotenuse + + using ::pow; + using ::powf; + using ::powl; + + using ::sqrt; + using ::sqrtf; + using ::sqrtl; + + using ::erf; + using ::erff; + using ::erfl; + + using ::erfc; + using ::erfcf; + using ::erfcl; + + using ::lgamma; + using ::lgammaf; + using ::lgammal; + + using ::tgamma; + using ::tgammaf; + using ::tgammal; + + using ::ceil; + using ::ceilf; + using ::ceill; + + using ::floor; + using ::floorf; + using ::floorl; + + using ::nearbyint; + using ::nearbyintf; + using ::nearbyintl; + + using ::rint; + using ::rintf; + using ::rintl; + + using ::lrint; + using ::lrintf; + using ::lrintl; + + using ::llrint; + using ::llrintf; + using ::llrintl; + + using ::round; + using ::roundf; + using ::roundl; + + using ::lround; + using ::lroundf; + using ::lroundl; + + using ::llround; + using ::llroundf; + using ::llroundl; + + using ::trunc; + using ::truncf; + using ::truncl; + + using ::fmod; + using ::fmodf; + using ::fmodl; + + using ::remainder; + using ::remainderf; + using ::remainderl; + + using ::remquo; + using ::remquof; + using ::remquol; + + using ::copysign; + using ::copysignf; + using ::copysignl; + + using ::nan; + using ::nanf; + using ::nanl; + + using ::nextafter; + using ::nextafterf; + using ::nextafterl; + + using ::nexttoward; + using ::nexttowardf; + using ::nexttowardl; + + using ::fdim; + using ::fdimf; + using ::fdiml; + + using ::fmax; + using ::fmaxf; + using ::fmaxl; + + using ::fmin; + using ::fminf; + using ::fminl; + + using ::fma; + using ::fmaf; + using ::fmal; + + // [c.math.lerp], linear interpolation + // [support.c.headers.other]/1 + // ... placed within the global namespace scope, except for the functions + // described in [sf.cmath], the std::lerp function overloads ([c.math.lerp]) + // ... + + // [c.math.fpclass], classification / comparison functions + using ::fpclassify; + using ::isfinite; + using ::isgreater; + using ::isgreaterequal; + using ::isinf; + using ::isless; + using ::islessequal; + using ::islessgreater; + using ::isnan; + using ::isnormal; + using ::isunordered; + using ::signbit; + + // [sf.cmath], mathematical special functions +} // export diff --git a/libcxx/modules/std.compat/csetjmp.inc b/libcxx/modules/std.compat/csetjmp.inc new file mode 100644 index 000000000000..1fc42ea3ee03 --- /dev/null +++ b/libcxx/modules/std.compat/csetjmp.inc @@ -0,0 +1,13 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::jmp_buf; + using ::longjmp; +} // export diff --git a/libcxx/modules/std.compat/csignal.inc b/libcxx/modules/std.compat/csignal.inc new file mode 100644 index 000000000000..33af6a9f2b73 --- /dev/null +++ b/libcxx/modules/std.compat/csignal.inc @@ -0,0 +1,17 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::sig_atomic_t; + + // [support.signal], signal handlers + using ::signal; + + using ::raise; +} // export diff --git a/libcxx/modules/std.compat/cstdarg.inc b/libcxx/modules/std.compat/cstdarg.inc new file mode 100644 index 000000000000..3efb34617a8b --- /dev/null +++ b/libcxx/modules/std.compat/cstdarg.inc @@ -0,0 +1,10 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { using ::va_list; } // export diff --git a/libcxx/modules/std.compat/cstddef.inc b/libcxx/modules/std.compat/cstddef.inc new file mode 100644 index 000000000000..94ad036fd8f4 --- /dev/null +++ b/libcxx/modules/std.compat/cstddef.inc @@ -0,0 +1,22 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::max_align_t; + using ::nullptr_t; + using ::ptrdiff_t; + using ::size_t; + + // [support.c.headers]/1 + // ... placed within the global namespace scope, except for ... the + // declaration of std::byte ([cstddef.syn]), and the functions and + // function templates described in [support.types.byteops]. ... + + // [support.types.byteops], byte type operations +} // export diff --git a/libcxx/modules/std.compat/cstdint.inc b/libcxx/modules/std.compat/cstdint.inc new file mode 100644 index 000000000000..1a74efc70cea --- /dev/null +++ b/libcxx/modules/std.compat/cstdint.inc @@ -0,0 +1,50 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // signed + using ::int8_t _LIBCPP_USING_IF_EXISTS; + using ::int16_t _LIBCPP_USING_IF_EXISTS; + using ::int32_t _LIBCPP_USING_IF_EXISTS; + using ::int64_t _LIBCPP_USING_IF_EXISTS; + + using ::int_fast16_t; + using ::int_fast32_t; + using ::int_fast64_t; + using ::int_fast8_t; + + using ::int_least16_t; + using ::int_least32_t; + using ::int_least64_t; + using ::int_least8_t; + + using ::intmax_t; + + using ::intptr_t _LIBCPP_USING_IF_EXISTS; + + // unsigned + using ::uint8_t _LIBCPP_USING_IF_EXISTS; + using ::uint16_t _LIBCPP_USING_IF_EXISTS; + using ::uint32_t _LIBCPP_USING_IF_EXISTS; + using ::uint64_t _LIBCPP_USING_IF_EXISTS; + + using ::uint_fast16_t; + using ::uint_fast32_t; + using ::uint_fast64_t; + using ::uint_fast8_t; + + using ::uint_least16_t; + using ::uint_least32_t; + using ::uint_least64_t; + using ::uint_least8_t; + + using ::uintmax_t; + + using ::uintptr_t _LIBCPP_USING_IF_EXISTS; +} // export diff --git a/libcxx/modules/std.compat/cstdio.inc b/libcxx/modules/std.compat/cstdio.inc new file mode 100644 index 000000000000..1ec3015c9e2a --- /dev/null +++ b/libcxx/modules/std.compat/cstdio.inc @@ -0,0 +1,61 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::FILE; + using ::fpos_t; + using ::size_t; + + using ::clearerr; + using ::fclose; + using ::feof; + using ::ferror; + using ::fflush; + using ::fgetc; + using ::fgetpos; + using ::fgets; + using ::fopen; + using ::fprintf; + using ::fputc; + using ::fputs; + using ::fread; + using ::freopen; + using ::fscanf; + using ::fseek; + using ::fsetpos; + using ::ftell; + using ::fwrite; + using ::getc; + using ::getchar; + using ::perror; + using ::printf; + using ::putc; + using ::putchar; + using ::puts; + using ::remove; + using ::rename; + using ::rewind; + using ::scanf; + using ::setbuf; + using ::setvbuf; + using ::snprintf; + using ::sprintf; + using ::sscanf; + using ::tmpfile; + using ::tmpnam; + using ::ungetc; + using ::vfprintf; + using ::vfscanf; + using ::vprintf; + using ::vscanf; + using ::vsnprintf; + using ::vsprintf; + using ::vsscanf; + +} // export diff --git a/libcxx/modules/std.compat/cstdlib.inc b/libcxx/modules/std.compat/cstdlib.inc new file mode 100644 index 000000000000..9333d8487071 --- /dev/null +++ b/libcxx/modules/std.compat/cstdlib.inc @@ -0,0 +1,72 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::div_t; + using ::ldiv_t; + using ::lldiv_t; + using ::size_t; + + // [support.start.term], start and termination + using ::_Exit; + using ::abort; + using ::at_quick_exit; + using ::atexit; + using ::exit; + using ::quick_exit; + + using ::getenv; + using ::system; + + // [c.malloc], C library memory allocation + using ::aligned_alloc; + using ::calloc; + using ::free; + using ::malloc; + using ::realloc; + + using ::atof; + using ::atoi; + using ::atol; + using ::atoll; + using ::strtod; + using ::strtof; + using ::strtol; + using ::strtold; + using ::strtoll; + using ::strtoul; + using ::strtoull; + + // [c.mb.wcs], multibyte / wide string and character conversion functions + using ::mblen; +#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS + using ::mbstowcs; + using ::mbtowc; + using ::wcstombs; + using ::wctomb; +#endif + // [alg.c.library], C standard library algorithms + using ::bsearch; + using ::qsort; + + // [c.math.rand], low-quality random number generation + using ::rand; + using ::srand; + + // [c.math.abs], absolute values + using ::abs; + + using ::labs; + using ::llabs; + + using ::div; + using ::ldiv; + using ::lldiv; + +} // export diff --git a/libcxx/modules/std.compat/cstring.inc b/libcxx/modules/std.compat/cstring.inc new file mode 100644 index 000000000000..090350ae8147 --- /dev/null +++ b/libcxx/modules/std.compat/cstring.inc @@ -0,0 +1,36 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::size_t; + + using ::memchr; + using ::memcmp; + using ::memcpy; + using ::memmove; + using ::memset; + using ::strcat; + using ::strchr; + using ::strcmp; + using ::strcoll; + using ::strcpy; + using ::strcspn; + using ::strerror; + using ::strlen; + using ::strncat; + using ::strncmp; + using ::strncpy; + using ::strpbrk; + using ::strrchr; + using ::strspn; + using ::strstr; + using ::strtok; + using ::strxfrm; + +} // export diff --git a/libcxx/modules/std.compat/ctime.inc b/libcxx/modules/std.compat/ctime.inc new file mode 100644 index 000000000000..92e3403a5e58 --- /dev/null +++ b/libcxx/modules/std.compat/ctime.inc @@ -0,0 +1,28 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + using ::clock_t; + using ::size_t; + using ::time_t; + + using ::timespec; + using ::tm; + + using ::asctime; + using ::clock; + using ::ctime; + using ::difftime; + using ::gmtime; + using ::localtime; + using ::mktime; + using ::strftime; + using ::time; + using ::timespec_get; +} // export diff --git a/libcxx/modules/std.compat/cuchar.inc b/libcxx/modules/std.compat/cuchar.inc new file mode 100644 index 000000000000..d1a511cadef1 --- /dev/null +++ b/libcxx/modules/std.compat/cuchar.inc @@ -0,0 +1,28 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { + // Note the Standard does not mark these symbols optional, but libc++'s header + // does. So this seems strictly not to be conforming. + + // mbstate_t is conditionally here, but always present in cwchar.cppm. To avoid + // conflicing declarations omit the using here. + + // size_t is conditionally here, but always present in cstddef.cppm. To avoid + // conflicing declarations omit the using here. + +#if !defined(_LIBCPP_HAS_NO_C8RTOMB_MBRTOC8) + using ::mbrtoc8 _LIBCPP_USING_IF_EXISTS; + using ::c8rtomb _LIBCPP_USING_IF_EXISTS; +#endif + using ::mbrtoc16 _LIBCPP_USING_IF_EXISTS; + using ::c16rtomb _LIBCPP_USING_IF_EXISTS; + using ::mbrtoc32 _LIBCPP_USING_IF_EXISTS; + using ::c32rtomb _LIBCPP_USING_IF_EXISTS; +} // export diff --git a/libcxx/modules/std.compat/cwchar.inc b/libcxx/modules/std.compat/cwchar.inc new file mode 100644 index 000000000000..8905aecbdfec --- /dev/null +++ b/libcxx/modules/std.compat/cwchar.inc @@ -0,0 +1,80 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { +#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS + using ::mbstate_t; + using ::size_t; + using ::wint_t; + + using ::tm; + + using ::btowc; + using ::fgetwc; + using ::fgetws; + using ::fputwc; + using ::fputws; + using ::fwide; + using ::fwprintf; + using ::fwscanf; + using ::getwc; + using ::getwchar; + using ::putwc; + using ::putwchar; + using ::swprintf; + using ::swscanf; + using ::ungetwc; + using ::vfwprintf; + using ::vfwscanf; + using ::vswprintf; + using ::vswscanf; + using ::vwprintf; + using ::vwscanf; + using ::wcscat; + using ::wcschr; + using ::wcscmp; + using ::wcscoll; + using ::wcscpy; + using ::wcscspn; + using ::wcsftime; + using ::wcslen; + using ::wcsncat; + using ::wcsncmp; + using ::wcsncpy; + using ::wcspbrk; + using ::wcsrchr; + using ::wcsspn; + using ::wcsstr; + using ::wcstod; + using ::wcstof; + using ::wcstok; + using ::wcstol; + using ::wcstold; + using ::wcstoll; + using ::wcstoul; + using ::wcstoull; + using ::wcsxfrm; + using ::wctob; + using ::wmemchr; + using ::wmemcmp; + using ::wmemcpy; + using ::wmemmove; + using ::wmemset; + using ::wprintf; + using ::wscanf; + + // [c.mb.wcs], multibyte / wide string and character conversion functions + using ::mbrlen; + using ::mbrtowc; + using ::mbsinit; + using ::mbsrtowcs; + using ::wcrtomb; + using ::wcsrtombs; +#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS +} // export diff --git a/libcxx/modules/std.compat/cwctype.inc b/libcxx/modules/std.compat/cwctype.inc new file mode 100644 index 000000000000..13aa2b7f3fb7 --- /dev/null +++ b/libcxx/modules/std.compat/cwctype.inc @@ -0,0 +1,35 @@ +// -*- C++ -*- +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +export { +#ifndef _LIBCPP_HAS_NO_WIDE_CHARACTERS + using ::wctrans_t; + using ::wctype_t; + using ::wint_t; + + using ::iswalnum; + using ::iswalpha; + using ::iswblank; + using ::iswcntrl; + using ::iswctype; + using ::iswdigit; + using ::iswgraph; + using ::iswlower; + using ::iswprint; + using ::iswpunct; + using ::iswspace; + using ::iswupper; + using ::iswxdigit; + using ::towctrans; + using ::towlower; + using ::towupper; + using ::wctrans; + using ::wctype; +#endif // _LIBCPP_HAS_NO_WIDE_CHARACTERS +} // export diff --git a/libcxx/modules/std.cppm.in b/libcxx/modules/std.cppm.in index ecb060126032..b46c52e781f8 100644 --- a/libcxx/modules/std.cppm.in +++ b/libcxx/modules/std.cppm.in @@ -8,7 +8,7 @@ //===----------------------------------------------------------------------===// // WARNING, this entire header is generated by -// utils/generate_std_cppm_in.py +// utils/generate_libcxx_cppm_in.py // DO NOT MODIFY! module; @@ -169,38 +169,38 @@ module; // *** Headers not yet available *** #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() #if __has_include() -# error "update the header information for in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include() +# error "please update the header information for in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include() export module std; diff --git a/libcxx/test/libcxx/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp b/libcxx/test/libcxx/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp index 42a7c08fa75b..a59839ff806e 100644 --- a/libcxx/test/libcxx/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp +++ b/libcxx/test/libcxx/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp @@ -45,7 +45,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/libcxx/module_std.gen.py b/libcxx/test/libcxx/module_std.gen.py index d8198cc70583..8e03d6e5b5b5 100644 --- a/libcxx/test/libcxx/module_std.gen.py +++ b/libcxx/test/libcxx/module_std.gen.py @@ -21,241 +21,17 @@ import sys sys.path.append(sys.argv[1]) -from libcxx.header_information import module_headers -from libcxx.header_information import header_restrictions - -BLOCKLIT = ( - "" # block Lit from interpreting a RUN/XFAIL/etc inside the generation script +from libcxx.test.modules import module_test_generator + +generator = module_test_generator( + "%t", + "%{module}", + "%{clang-tidy}", + "%{test-tools}/clang_tidy_checks/libcxx-tidy.plugin", + "%{cxx}", + "%{flags} %{compile_flags}", ) -# Ignore several declarations found in the includes. -# -# Part of these items are bugs other are not yet implemented features. -SkipDeclarations = dict() - -# See comment in the header. -SkipDeclarations["cuchar"] = ["std::mbstate_t", "std::size_t"] - -# Not in the synopsis. -SkipDeclarations["cwchar"] = ["std::FILE"] - -# The operators are added for private types like __iom_t10. -SkipDeclarations["iomanip"] = ["std::operator<<", "std::operator>>"] - -SkipDeclarations["iosfwd"] = ["std::ios_base", "std::vector"] - -# This header also provides declarations in the namespace that might be -# an error. -SkipDeclarations["filesystem"] = [ - "std::filesystem::operator==", - "std::filesystem::operator!=", -] - -# This is a specialization for a private type -SkipDeclarations["iterator"] = ["std::pointer_traits"] - -# TODO MODULES -# This definition is declared in string and defined in istream -# This declaration should be part of string -SkipDeclarations["istream"] = ["std::getline"] - -# P1614 (at many places) and LWG3519 too. -SkipDeclarations["random"] = [ - "std::operator!=", - # LWG3519 makes these hidden friends. - # Note the older versions had the requirement of these operations but not in - # the synopsis. - "std::operator<<", - "std::operator>>", - "std::operator==", -] - -# Declared in the forward header since std::string uses std::allocator -SkipDeclarations["string"] = ["std::allocator"] -# TODO MODULES remove zombie names -# https://libcxx.llvm.org/Status/Cxx20.html#note-p0619 -SkipDeclarations["memory"] = [ - "std::return_temporary_buffer", - "std::get_temporary_buffer", -] - -# TODO MODULES this should be part of ios instead -SkipDeclarations["streambuf"] = ["std::basic_ios"] - -# include/__type_traits/is_swappable.h -SkipDeclarations["type_traits"] = [ - "std::swap", - # TODO MODULES gotten through __functional/unwrap_ref.h - "std::reference_wrapper", -] - -# Add declarations in headers. -# -# Some headers have their defines in a different header, which may have -# additional declarations. -ExtraDeclarations = dict() -# This declaration is in the ostream header. -ExtraDeclarations["system_error"] = ["std::operator<<"] - -# Adds an extra header file to scan -# -# -ExtraHeader = dict() -# locale has a file and not a subdirectory -ExtraHeader["locale"] = "v1/__locale$" -ExtraHeader["thread"] = "v1/__threading_support$" -ExtraHeader["ranges"] = "v1/__fwd/subrange.h$" - -# The extra header is needed since two headers are required to provide the -# same definition. -ExtraHeader["functional"] = "v1/__compare/compare_three_way.h$" - -# newline needs to be escaped for the module partition output. -nl = '\\\\n' - -# Create empty file with all parts. -print( - f"""\ -//--- module_std.sh.cpp -// UNSUPPORTED{BLOCKLIT}: c++03, c++11, c++14, c++17 -// UNSUPPORTED{BLOCKLIT}: libcpp-has-no-std-modules -// UNSUPPORTED{BLOCKLIT}: clang-modules-build - -// REQUIRES{BLOCKLIT}: has-clang-tidy - -// The GCC compiler flags are not always compatible with clang-tidy. -// UNSUPPORTED{BLOCKLIT}: gcc - -// RUN{BLOCKLIT}: echo -n > %t.all_partitions -""" -) - -# Validate all module parts. -for header in module_headers: - # Some headers cannot be included when a libc++ feature is disabled. - # In that case include the header conditionally. The header __config - # ensures the libc++ feature macros are available. - if header in header_restrictions: - include = ( - f"#include <__config>{nl}" - + f"#if {header_restrictions[header]}{nl}" - + f"# include <{header}>{nl}" - + f"#endif{nl}" - ) - elif header == "chrono": - # When localization is disabled the header string is not included. - # When string is included chrono's operator""s is a named declaration - # using std::chrono_literals::operator""s; - # else it is a named declaration - # using std::operator""s; - # TODO MODULES investigate why - include = f"#include {nl}#include {nl}" - else: - include = f"#include <{header}>{nl}" - - # Generate a module partition for the header module includes. This - # makes it possible to verify that all headers export all their - # named declarations. - print( - f"// RUN{BLOCKLIT}: echo -e \"" - f"module;{nl}" - f"{include}" - f"{nl}" - f"// Use __libcpp_module_
to ensure that modules {nl}" - f"// are not named as keywords or reserved names.{nl}" - f"export module std:__libcpp_module_{header};{nl}" - f'#include \\"%{{module}}/std/{header}.inc\\"{nl}' - f"\" > %t.{header}.cppm") - - # Dump the information as found in the module's cppm file. - print( - f"// RUN{BLOCKLIT}: %{{clang-tidy}} %t.{header}.cppm " - " --checks='-*,libcpp-header-exportable-declarations' " - " -config='{CheckOptions: [ " - " {" - " key: libcpp-header-exportable-declarations.Filename, " - f" value: {header}.inc" - " }, {" - " key: libcpp-header-exportable-declarations.FileType, " - " value: ModulePartition" - " }, " - " ]}' " - " --load=%{test-tools}/clang_tidy_checks/libcxx-tidy.plugin " - " -- %{flags} %{compile_flags} " - f"| sort > %t.{header}.module" - ) - print(f"// RUN{BLOCKLIT}: cat %t.{header}.module >> %t.all_partitions") - - # Dump the information as found in the module by using the header file(s). - skip_declarations = " ".join(SkipDeclarations.get(header, [])) - if skip_declarations: - skip_declarations = ( - "{" - " key: libcpp-header-exportable-declarations.SkipDeclarations, " - f' value: "{skip_declarations}" ' - "}, " - ) - - extra_declarations = " ".join(ExtraDeclarations.get(header, [])) - if extra_declarations: - extra_declarations = ( - " {" - " key: libcpp-header-exportable-declarations.ExtraDeclarations, " - f' value: "{extra_declarations}" ' - "}, " - ) - - extra_header = ExtraHeader.get(header, "") - if extra_header: - extra_header = ( - "{" - " key: libcpp-header-exportable-declarations.ExtraHeader, " - f' value: "{extra_header}" ' - "}, " - ) - - # Clang-tidy needs a file input - print(f'// RUN{BLOCKLIT}: echo -e "' f"{include}" f'" > %t.{header}.cpp') - print( - f"// RUN{BLOCKLIT}: %{{clang-tidy}} %t.{header}.cpp " - " --checks='-*,libcpp-header-exportable-declarations' " - " -config='{CheckOptions: [ " - f" {{key: libcpp-header-exportable-declarations.Filename, value: {header}}}, " - " {key: libcpp-header-exportable-declarations.FileType, value: Header}, " - f" {skip_declarations} {extra_declarations} {extra_header}, " - " ]}' " - " --load=%{test-tools}/clang_tidy_checks/libcxx-tidy.plugin " - " -- %{flags} %{compile_flags} " - f" | sort > %t.{header}.include" - ) - - # Compare the cppm and header file(s) return the same results. - print(f"// RUN{BLOCKLIT}: diff -u %t.{header}.module %t.{header}.include") - - -# Merge the data of the parts -print(f"// RUN{BLOCKLIT}: sort -u -o %t.all_partitions %t.all_partitions") - -# Dump the information as found in std.cppm. -print( - f"// RUN{BLOCKLIT}: %{{clang-tidy}} %{{module}}/std.cppm " - " --checks='-*,libcpp-header-exportable-declarations' " - " -config='{CheckOptions: [ " - " {key: libcpp-header-exportable-declarations.Header, value: std.cppm}, " - " {key: libcpp-header-exportable-declarations.FileType, value: Module}, " - " ]}' " - f" --load=%{{test-tools}}/clang_tidy_checks/libcxx-tidy.plugin " - " -- %{flags} %{compile_flags} " - " | sort > %t.module" -) - - -# Compare the sum of the parts with the main module. -print(f"// RUN{BLOCKLIT}: diff -u %t.all_partitions %t.module") -# Basic smoke test. Import a module and try to compile when using all -# exported names. This validates the clang-tidy script does not accidentally -# add named declarations to the list that are not available. -print(f"// RUN{BLOCKLIT}: echo 'import std;' > %t.compile.pass.cpp") -print(f"// RUN{BLOCKLIT}: cat %t.all_partitions >> %t.compile.pass.cpp") -print(f"// RUN{BLOCKLIT}: %{{cxx}} %{{flags}} %{{compile_flags}} -fsyntax-only %t.compile.pass.cpp") +print("//--- module_std.sh.cpp") +generator.write_test("std") diff --git a/libcxx/test/libcxx/module_std_compat.gen.py b/libcxx/test/libcxx/module_std_compat.gen.py new file mode 100644 index 000000000000..c4792db3d71e --- /dev/null +++ b/libcxx/test/libcxx/module_std_compat.gen.py @@ -0,0 +1,62 @@ +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +# Test that all named declarations with external linkage match the +# exported declarations in their associated module partition. +# Then it tests the sum of the exported declarations in the module +# partitions matches the export of the std.compat module. + +# Note the test of the std.compat module requires all partitions to be tested +# first. Since lit tests have no dependencies, this means the test needs +# to be one monolitic test. Since the test doesn't take very long it's +# not a huge issue. + +# RUN: %{python} %s %{libcxx}/utils + +import sys + +sys.path.append(sys.argv[1]) +from libcxx.test.modules import module_test_generator + +generator = module_test_generator( + "%t", + "%{module}", + "%{clang-tidy}", + "%{test-tools}/clang_tidy_checks/libcxx-tidy.plugin", + "%{cxx}", + "%{flags} %{compile_flags}", +) + + +print("//--- module_std_compat.sh.cpp") +generator.write_test( + "std.compat", + [ + "cassert", + "cctype", + "cerrno", + "cfenv", + "cfloat", + "cinttypes", + "climits", + "clocale", + "cmath", + "csetjmp", + "csignal", + "cstdarg", + "cstddef", + "cstdint", + "cstdio", + "cstdlib", + "cstring", + "ctime", + "cuchar", + "cwchar", + "cwctype", + ], +) diff --git a/libcxx/test/libcxx/numerics/rand/rand.req.urng/valid_real_type.verify.cpp b/libcxx/test/libcxx/numerics/rand/rand.req.urng/valid_real_type.verify.cpp new file mode 100644 index 000000000000..6c6a109227df --- /dev/null +++ b/libcxx/test/libcxx/numerics/rand/rand.req.urng/valid_real_type.verify.cpp @@ -0,0 +1,109 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// + +#include + +void test() { + { + std::uniform_real_distribution + baddist; //expected-error@*:* {{RealType must be a supported floating-point type}} + std::uniform_real_distribution okdist; + (void)baddist; + (void)okdist; + } + { + std::exponential_distribution + baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::exponential_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::gamma_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::gamma_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::weibull_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::weibull_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::extreme_value_distribution + baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::extreme_value_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::normal_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::normal_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::lognormal_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::lognormal_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::chi_squared_distribution + baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::chi_squared_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::cauchy_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::cauchy_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::fisher_f_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::fisher_f_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::student_t_distribution baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::student_t_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::piecewise_constant_distribution + baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::piecewise_constant_distribution okdist; + (void)baddist; + (void)okdist; + } + + { + std::piecewise_linear_distribution + baddist; // expected-error@*:* {{RealType must be a supported floating-point type}} + std::piecewise_linear_distribution okdist; + (void)baddist; + (void)okdist; + } +} diff --git a/libcxx/test/libcxx/selftest/stdin-is-piped.sh.cpp b/libcxx/test/libcxx/selftest/stdin-is-piped.sh.cpp index 897e10e94783..ffd10631c6a6 100644 --- a/libcxx/test/libcxx/selftest/stdin-is-piped.sh.cpp +++ b/libcxx/test/libcxx/selftest/stdin-is-piped.sh.cpp @@ -8,8 +8,6 @@ // Make sure that the executor pipes standard input to the test-executable being run. -// XFAIL: LIBCXX-PICOLIBC-FIXME - // RUN: %{build} // RUN: echo "abc" | %{exec} %t.exe diff --git a/libcxx/test/lit.local.cfg b/libcxx/test/lit.local.cfg index 4116553b6f7a..1ee9086ee22e 100644 --- a/libcxx/test/lit.local.cfg +++ b/libcxx/test/lit.local.cfg @@ -67,3 +67,17 @@ if ( "%{link_flags}", os.path.join(build, "libc++std.a"), ) + + config.substitutions = appendToSubstitution( + config.substitutions, + "%{compile_flags}", + "-fprebuilt-module-path=" + + os.path.join( + config.test_exec_root, "__config_module__/CMakeFiles/std.compat.dir" + ), + ) + config.substitutions = appendToSubstitution( + config.substitutions, + "%{link_flags}", + os.path.join(build, "libc++std.compat.a"), + ) diff --git a/libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/ranges_unique_copy.pass.cpp b/libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/ranges_unique_copy.pass.cpp index b84600b92c2b..a4cf97069c96 100644 --- a/libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/ranges_unique_copy.pass.cpp +++ b/libcxx/test/std/algorithms/alg.modifying.operations/alg.unique/ranges_unique_copy.pass.cpp @@ -418,7 +418,7 @@ constexpr bool test() { assert(std::ranges::equal(out, expected)); assert(base(result.in) == in.end()); assert(base(result.out) == out.end()); - assert(numberOfComp == in.size() - 1); + assert(numberOfComp == static_cast(in.size() - 1)); assert(numberOfProj <= static_cast(2 * (in.size() - 1))); } // range overload @@ -434,7 +434,7 @@ constexpr bool test() { assert(std::ranges::equal(out, expected)); assert(base(result.in) == in.end()); assert(base(result.out) == out.end()); - assert(numberOfComp == in.size() - 1); + assert(numberOfComp == static_cast(in.size() - 1)); assert(numberOfProj <= static_cast(2 * (in.size() - 1))); } } diff --git a/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/sort.heap/ranges_sort_heap.pass.cpp b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/sort.heap/ranges_sort_heap.pass.cpp index 1153ed573d63..1e636ea9afac 100644 --- a/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/sort.heap/ranges_sort_heap.pass.cpp +++ b/libcxx/test/std/algorithms/alg.sorting/alg.heap.operations/sort.heap/ranges_sort_heap.pass.cpp @@ -238,6 +238,7 @@ void test_complexity() { const int debug_elements = std::min(100, n); // Multiplier 2 because of comp(a,b) comp(b, a) checks. const int debug_comparisons = 2 * (debug_elements + 1) * debug_elements; + (void)debug_comparisons; std::shuffle(first, last, g); std::make_heap(first, last, &MyInt::Comp); // The exact stats of our current implementation are recorded here. @@ -247,7 +248,6 @@ void test_complexity() { LIBCPP_ASSERT(stats.moved <= 2 * n + n * logn); #if _LIBCPP_HARDENING_MODE != _LIBCPP_HARDENING_MODE_DEBUG LIBCPP_ASSERT(stats.compared <= n * logn); - (void)debug_comparisons; #else LIBCPP_ASSERT(stats.compared <= 2 * n * logn + debug_comparisons); #endif diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/assign.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/assign.pass.cpp index 66f6c91b802c..1561bd27d8d7 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/assign.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/assign.pass.cpp @@ -56,7 +56,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_strong.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_strong.pass.cpp index 711b49ddc233..84ef7b70bc0a 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_strong.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_strong.pass.cpp @@ -220,7 +220,7 @@ void test() { int main(int, char**) { test(); test(); - // https://github.com/llvm/llvm-project/issues/47978 + // TODO https://github.com/llvm/llvm-project/issues/47978 // test(); return 0; diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_weak.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_weak.pass.cpp index ee1a00c28427..54ca29a0d427 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_weak.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/compare_exchange_weak.pass.cpp @@ -68,7 +68,7 @@ void testBasic(MemoryOrder... memory_order) { assert(a.load() == T(1.2)); // bug - // https://github.com/llvm/llvm-project/issues/47978 + // TODO https://github.com/llvm/llvm-project/issues/47978 if constexpr (!std::same_as) { assert(expected == T(1.2)); } @@ -236,10 +236,8 @@ int main(int, char**) { test(); test(); -// https://github.com/llvm/llvm-project/issues/47978 -#ifndef TEST_COMPILER_CLANG - test(); -#endif + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/ctor.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/ctor.pass.cpp index aad635408476..92351d74f35c 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/ctor.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/ctor.pass.cpp @@ -56,7 +56,8 @@ constexpr void testOne() { constexpr bool test() { testOne(); testOne(); - testOne(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // testOne(); return true; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/exchange.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/exchange.pass.cpp index c0d1eb686db7..c365923c42d0 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/exchange.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/exchange.pass.cpp @@ -6,9 +6,6 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// Clang's support for atomic operations on long double is broken. See https://github.com/llvm/llvm-project/issues/72893 -// XFAIL: target={{x86_64-.*}} && tsan -// XFAIL: target={{x86_64-.*}} && msan // XFAIL: !has-64-bit-atomics // UNSUPPORTED: !non-lockfree-atomics @@ -72,7 +69,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp index c31243053e69..d74502f52e1b 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_add.pass.cpp @@ -7,13 +7,12 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 // UNSUPPORTED: LIBCXX-AIX-FIXME -// Clang's support for atomic operations on long double is broken. See https://github.com/llvm/llvm-project/issues/72893 -// XFAIL: target={{x86_64-.*}} && tsan -// Hangs with msan. -// UNSUPPORTED: msan // XFAIL: !has-64-bit-atomics // UNSUPPORTED: !non-lockfree-atomics +// https://github.com/llvm/llvm-project/issues/72893 +// XFAIL: target={{x86_64-.*}} && tsan + // floating-point-type fetch_add(floating-point-type, // memory_order = memory_order::seq_cst) volatile noexcept; // floating-point-type fetch_add(floating-point-type, @@ -114,7 +113,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp index d26d21dec1b3..2b07f1fdbb5b 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/fetch_sub.pass.cpp @@ -7,13 +7,12 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 // UNSUPPORTED: LIBCXX-AIX-FIXME -// Clang's support for atomic operations on long double is broken. See https://github.com/llvm/llvm-project/issues/72893 -// XFAIL: target={{x86_64-.*}} && tsan -// Hangs with msan. -// UNSUPPORTED: msan // XFAIL: !has-64-bit-atomics // UNSUPPORTED: !non-lockfree-atomics +// https://github.com/llvm/llvm-project/issues/72893 +// XFAIL: target={{x86_64-.*}} && tsan + // floating-point-type fetch_sub(floating-point-type, // memory_order = memory_order::seq_cst) volatile noexcept; // floating-point-type fetch_sub(floating-point-type, @@ -115,7 +114,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/load.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/load.pass.cpp index b567af457dca..784fdb96c464 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/load.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/load.pass.cpp @@ -6,9 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// Clang's support for atomic operations on long double is broken. See https://github.com/llvm/llvm-project/issues/72893 // XFAIL: !has-64-bit-atomics -// XFAIL: target={{x86_64-.*}} && tsan // UNSUPPORTED: !non-lockfree-atomics // floating-point-type load(memory_order = memory_order::seq_cst) volatile noexcept; @@ -134,7 +132,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp index b5422c0362dc..cf5af00cd4e8 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/lockfree.pass.cpp @@ -55,7 +55,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_all.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_all.pass.cpp index b50c0ad9ee92..d8bec9160c68 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_all.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_all.pass.cpp @@ -93,7 +93,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_one.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_one.pass.cpp index aee4af186b9f..fd4ea44ff851 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_one.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/notify_one.pass.cpp @@ -77,7 +77,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.float.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.float.pass.cpp index 5d957ca68404..a1733276c0ec 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.float.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.float.pass.cpp @@ -52,7 +52,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp index 269eb819524f..22233b5a62ba 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.minus_equals.pass.cpp @@ -6,11 +6,9 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: !has-64-bit-atomics // UNSUPPORTED: LIBCXX-AIX-FIXME +// XFAIL: !has-64-bit-atomics // UNSUPPORTED: !non-lockfree-atomics -// Hangs with msan. -// UNSUPPORTED: msan // floating-point-type operator-=(floating-point-type) volatile noexcept; // floating-point-type operator-=(floating-point-type) noexcept; @@ -98,7 +96,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp index 9a2298ea46d6..548c5ac11a26 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/operator.plus_equals.pass.cpp @@ -6,11 +6,9 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// XFAIL: !has-64-bit-atomics // UNSUPPORTED: LIBCXX-AIX-FIXME +// XFAIL: !has-64-bit-atomics // UNSUPPORTED: !non-lockfree-atomics -// Hangs with msan. -// UNSUPPORTED: msan // floating-point-type operator+=(floating-point-type) volatile noexcept; // floating-point-type operator+=(floating-point-type) noexcept; @@ -98,7 +96,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/store.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/store.pass.cpp index b0888aac67db..d232eea6ef01 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/store.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/store.pass.cpp @@ -6,9 +6,7 @@ // //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 -// Clang's support for atomic operations on long double is broken. See https://github.com/llvm/llvm-project/issues/72893 // XFAIL: !has-64-bit-atomics -// XFAIL: target={{x86_64-.*}} && tsan // UNSUPPORTED: !non-lockfree-atomics // void store(floating-point-type, memory_order = memory_order::seq_cst) volatile noexcept; @@ -109,7 +107,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/wait.pass.cpp b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/wait.pass.cpp index 8e003a7e365d..ad70c133b99e 100644 --- a/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/wait.pass.cpp +++ b/libcxx/test/std/atomics/atomics.types.generic/atomics.types.float/wait.pass.cpp @@ -7,9 +7,6 @@ //===----------------------------------------------------------------------===// // UNSUPPORTED: c++03, c++11, c++14, c++17 // XFAIL: availability-synchronization_library-missing -// Clang's support for atomic operations on long double is broken. See https://github.com/llvm/llvm-project/issues/72893 -// XFAIL: target={{x86_64-.*}} && tsan -// XFAIL: target={{x86_64-.*}} && msan // XFAIL: !has-64-bit-atomics // UNSUPPORTED: !non-lockfree-atomics @@ -120,7 +117,8 @@ void test() { int main(int, char**) { test(); test(); - test(); + // TODO https://github.com/llvm/llvm-project/issues/47978 + // test(); return 0; } diff --git a/libcxx/test/std/containers/sequences/insert_range_sequence_containers.h b/libcxx/test/std/containers/sequences/insert_range_sequence_containers.h index b77b2510eae2..352ee474cae7 100644 --- a/libcxx/test/std/containers/sequences/insert_range_sequence_containers.h +++ b/libcxx/test/std/containers/sequences/insert_range_sequence_containers.h @@ -427,7 +427,8 @@ template <> constexpr TestCase FullContainer_End_LongRange { template constexpr void test_sequence_insert_range(Validate validate) { using T = typename Container::value_type; - auto get_pos = [](auto& c, auto& test_case) { return std::ranges::next(c.begin(), test_case.index); }; + using D = typename Container::difference_type; + auto get_pos = [](auto& c, auto& test_case) { return std::ranges::next(c.begin(), static_cast(test_case.index)); }; auto test = [&](auto& test_case) { Container c(test_case.initial.begin(), test_case.initial.end()); diff --git a/libcxx/test/std/containers/unord/unord.map/eq.different_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.map/eq.different_hash.pass.cpp index 52a806f04d72..94de7589eb06 100644 --- a/libcxx/test/std/containers/unord/unord.map/eq.different_hash.pass.cpp +++ b/libcxx/test/std/containers/unord/unord.map/eq.different_hash.pass.cpp @@ -36,7 +36,7 @@ std::size_t hash_neg(T val) { } template std::size_t hash_scale(T val) { - return val << 1; + return static_cast(val << 1); } template std::size_t hash_even(T val) { @@ -57,7 +57,7 @@ std::size_t hash_neg(T* val) { } template std::size_t hash_scale(T* val) { - return *val << 1; + return static_cast(*val << 1); } template std::size_t hash_even(T* val) { diff --git a/libcxx/test/std/containers/unord/unord.multimap/eq.different_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.multimap/eq.different_hash.pass.cpp index 2644c9657644..0aafb401d42c 100644 --- a/libcxx/test/std/containers/unord/unord.multimap/eq.different_hash.pass.cpp +++ b/libcxx/test/std/containers/unord/unord.multimap/eq.different_hash.pass.cpp @@ -37,7 +37,7 @@ std::size_t hash_neg(T val) { } template std::size_t hash_scale(T val) { - return val << 1; + return static_cast(val << 1); } template std::size_t hash_even(T val) { @@ -58,7 +58,7 @@ std::size_t hash_neg(T* val) { } template std::size_t hash_scale(T* val) { - return *val << 1; + return static_cast(*val << 1); } template std::size_t hash_even(T* val) { diff --git a/libcxx/test/std/containers/unord/unord.multiset/eq.different_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.multiset/eq.different_hash.pass.cpp index 9f53e8d79e86..5b8f11e92927 100644 --- a/libcxx/test/std/containers/unord/unord.multiset/eq.different_hash.pass.cpp +++ b/libcxx/test/std/containers/unord/unord.multiset/eq.different_hash.pass.cpp @@ -36,7 +36,7 @@ std::size_t hash_neg(T val) { } template std::size_t hash_scale(T val) { - return val << 1; + return static_cast(val << 1); } template std::size_t hash_even(T val) { @@ -57,7 +57,7 @@ std::size_t hash_neg(T* val) { } template std::size_t hash_scale(T* val) { - return *val << 1; + return static_cast(*val << 1); } template std::size_t hash_even(T* val) { diff --git a/libcxx/test/std/containers/unord/unord.set/eq.different_hash.pass.cpp b/libcxx/test/std/containers/unord/unord.set/eq.different_hash.pass.cpp index a763c7fee623..3cb4815a5bcb 100644 --- a/libcxx/test/std/containers/unord/unord.set/eq.different_hash.pass.cpp +++ b/libcxx/test/std/containers/unord/unord.set/eq.different_hash.pass.cpp @@ -36,7 +36,7 @@ std::size_t hash_neg(T val) { } template std::size_t hash_scale(T val) { - return val << 1; + return static_cast(val << 1); } template std::size_t hash_even(T val) { @@ -56,11 +56,11 @@ std::size_t hash_neg(T* val) { return std::numeric_limits::max() - *val; } template -size_t hash_scale(T* val) { - return *val << 1; +std::size_t hash_scale(T* val) { + return static_cast(*val << 1); } template -size_t hash_even(T* val) { +std::size_t hash_even(T* val) { return *val & 1 ? 1 : 0; } diff --git a/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp b/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp index 01278e907671..30281a8d922d 100644 --- a/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/layout_stride/index_operator.pass.cpp @@ -53,10 +53,10 @@ constexpr void iterate_stride(M m, const std::array(M::extents_type::rank()) - 1 - static_cast(sizeof...(Args)); if constexpr (-1 == r) { ASSERT_NOEXCEPT(m(args...)); - size_t expected_val = [&](std::index_sequence) { + std::size_t expected_val = static_cast([&](std::index_sequence) { return ((args * strides[Pos]) + ... + 0); - }(std::make_index_sequence()); - assert(expected_val == static_cast(m(args...))); + }(std::make_index_sequence())); + assert(expected_val == static_cast(m(args...))); } else { for (typename M::index_type i = 0; i < m.extents().extent(r); i++) { iterate_stride(m, strides, i, args...); @@ -73,7 +73,7 @@ constexpr void test_iteration(std::array strides, Args... args) } constexpr bool test() { - constexpr size_t D = std::dynamic_extent; + constexpr std::size_t D = std::dynamic_extent; test_iteration>(std::array{}); test_iteration>(std::array{2}, 1); test_iteration>(std::array{3}, 7); @@ -102,7 +102,7 @@ constexpr bool test() { } constexpr bool test_large() { - constexpr size_t D = std::dynamic_extent; + constexpr std::size_t D = std::dynamic_extent; test_iteration>(std::array{2000, 2, 20, 200}, 7, 9, 10); test_iteration>(std::array{2000, 20, 20, 200}, 7, 10); return true; diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp index bbf8137a7cb8..0e0c7667da30 100644 --- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_array.pass.cpp @@ -65,7 +65,7 @@ test_mdspan_ctor_array(const H& handle, const M& map, const A&, std::array == (N == MDS::rank_dynamic())); diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp index 0a8918028d13..40e82db98635 100644 --- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_extents.pass.cpp @@ -51,7 +51,7 @@ constexpr void test_mdspan_types(const H& handle, const M& map, const A&) { assert((H::move_counter() == 1)); } } - static_assert(!noexcept(MDS(handle, map.extents()))); + LIBCPP_STATIC_ASSERT(!noexcept(MDS(handle, map.extents()))); assert(m.extents() == map.extents()); if constexpr (std::equality_comparable) assert(m.data_handle() == handle); diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp index 75b7722b445e..fa65848ac69b 100644 --- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map.pass.cpp @@ -48,7 +48,7 @@ constexpr void test_mdspan_types(const H& handle, const M& map, const A&) { assert((H::move_counter() == 1)); } } - static_assert(!noexcept(MDS(handle, map))); + LIBCPP_STATIC_ASSERT(!noexcept(MDS(handle, map))); assert(m.extents() == map.extents()); if constexpr (std::equality_comparable) assert(m.data_handle() == handle); diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp index c2210f1dba90..65d32f3d7a7f 100644 --- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_map_acc.pass.cpp @@ -43,7 +43,7 @@ constexpr void test_mdspan_types(const H& handle, const M& map, const A& acc) { assert((H::move_counter() == 1)); } } - static_assert(!noexcept(MDS(handle, map, acc))); + LIBCPP_STATIC_ASSERT(!noexcept(MDS(handle, map, acc))); assert(m.extents() == map.extents()); if constexpr (std::equality_comparable) assert(m.data_handle() == handle); diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp index 980619795b5d..f4fb5e681d95 100644 --- a/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/mdspan/ctor.dh_span.pass.cpp @@ -65,7 +65,7 @@ test_mdspan_ctor_span(const H& handle, const M& map, const A&, std::span == (N == MDS::rank_dynamic())); diff --git a/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp b/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp index 35534fa87954..ba1fef1df677 100644 --- a/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp +++ b/libcxx/test/std/containers/views/mdspan/mdspan/properties.pass.cpp @@ -27,11 +27,12 @@ // constexpr const data_handle_type& data_handle() const noexcept { return ptr_; } // constexpr const mapping_type& mapping() const noexcept { return map_; } // constexpr const accessor_type& accessor() const noexcept { return acc_; } -// static constexpr bool is_always_unique() +// /* per LWG-4021 "mdspan::is_always_meow() should be noexcept" */ +// static constexpr bool is_always_unique() noexcept // { return mapping_type::is_always_unique(); } -// static constexpr bool is_always_exhaustive() +// static constexpr bool is_always_exhaustive() noexcept // { return mapping_type::is_always_exhaustive(); } -// static constexpr bool is_always_strided() +// static constexpr bool is_always_strided() noexcept // { return mapping_type::is_always_strided(); } // // constexpr bool is_unique() const @@ -141,15 +142,16 @@ constexpr void test_mdspan_types(const H& handle, const M& map, const A& acc) { ASSERT_SAME_TYPE(decltype(m.is_unique()), bool); ASSERT_SAME_TYPE(decltype(m.is_exhaustive()), bool); ASSERT_SAME_TYPE(decltype(m.is_strided()), bool); - assert(!noexcept(MDS::is_always_unique())); - assert(!noexcept(MDS::is_always_exhaustive())); - assert(!noexcept(MDS::is_always_strided())); - assert(!noexcept(m.is_unique())); - assert(!noexcept(m.is_exhaustive())); - assert(!noexcept(m.is_strided())); - assert(MDS::is_always_unique() == M::is_always_unique()); - assert(MDS::is_always_exhaustive() == M::is_always_exhaustive()); - assert(MDS::is_always_strided() == M::is_always_strided()); + // per LWG-4021 "mdspan::is_always_meow() should be noexcept" + static_assert(noexcept(MDS::is_always_unique())); + static_assert(noexcept(MDS::is_always_exhaustive())); + static_assert(noexcept(MDS::is_always_strided())); + LIBCPP_STATIC_ASSERT(!noexcept(m.is_unique())); + LIBCPP_STATIC_ASSERT(!noexcept(m.is_exhaustive())); + LIBCPP_STATIC_ASSERT(!noexcept(m.is_strided())); + static_assert(MDS::is_always_unique() == M::is_always_unique()); + static_assert(MDS::is_always_exhaustive() == M::is_always_exhaustive()); + static_assert(MDS::is_always_strided() == M::is_always_strided()); assert(m.is_unique() == map.is_unique()); assert(m.is_exhaustive() == map.is_exhaustive()); assert(m.is_strided() == map.is_strided()); @@ -159,7 +161,7 @@ constexpr void test_mdspan_types(const H& handle, const M& map, const A& acc) { if (m.is_strided()) { for (typename MDS::rank_type r = 0; r < MDS::rank(); r++) { ASSERT_SAME_TYPE(decltype(m.stride(r)), typename MDS::index_type); - assert(!noexcept(m.stride(r))); + LIBCPP_STATIC_ASSERT(!noexcept(m.stride(r))); assert(m.stride(r) == map.stride(r)); } } diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/buffered_reads.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/buffered_reads.pass.cpp index d57b7c20a2da..ecc11f4999ff 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/buffered_reads.pass.cpp +++ b/libcxx/test/std/input.output/file.streams/fstreams/ifstream.members/buffered_reads.pass.cpp @@ -44,7 +44,7 @@ template void test_read(BufferPolicy policy, const std::vector& payload_sizes) { - std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), 0); + std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), std::streamsize{0}); std::vector data(total_size); for (std::size_t i = 0; i < data.size(); ++i) { data[i] = static_cast(i % (1 << 8 * sizeof(char))); @@ -99,7 +99,7 @@ void test_read(BufferPolicy policy, const std::vector& payload_ #ifndef TEST_HAS_NO_WIDE_CHARACTERS template void test_read_codecvt(BufferPolicy policy, const std::vector& payload_sizes) { - std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), 0); + std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), std::streamsize{0}); std::vector data(total_size); for (std::size_t i = 0; i < data.size(); ++i) { data[i] = static_cast(i); diff --git a/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/buffered_writes.pass.cpp b/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/buffered_writes.pass.cpp index e78207395051..b5bbb0ca2ee4 100644 --- a/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/buffered_writes.pass.cpp +++ b/libcxx/test/std/input.output/file.streams/fstreams/ofstream.members/buffered_writes.pass.cpp @@ -45,7 +45,7 @@ template void test_write(BufferPolicy policy, const std::vector& payload_sizes) { std::size_t previously_written = 0; - std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), 0); + std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), std::streamsize{0}); std::vector data(total_size); for (std::size_t i = 0; i < data.size(); ++i) { data[i] = static_cast(i % (1 << 8 * sizeof(char))); @@ -97,7 +97,7 @@ void test_write(BufferPolicy policy, const std::vector& payload template void test_write_codecvt(BufferPolicy policy, const std::vector& payload_sizes) { std::size_t previously_written = 0; - std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), 0); + std::streamsize total_size = std::accumulate(payload_sizes.begin(), payload_sizes.end(), std::streamsize{0}); std::vector data(total_size); for (std::size_t i = 0; i < data.size(); ++i) { data[i] = static_cast(i); diff --git a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.assign/source.pass.cpp b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.assign/source.pass.cpp index 893d09221fce..e969aa4e8d66 100644 --- a/libcxx/test/std/input.output/filesystems/class.path/path.member/path.assign/source.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/class.path/path.member/path.assign/source.pass.cpp @@ -230,7 +230,7 @@ void RunStringMoveTest(const fs::path::value_type* Expect) { assert(p == Expect); { // Signature test - LIBCPP_ONLY(ASSERT_NOEXCEPT(p = std::move(ss))); + LIBCPP_ASSERT_NOEXCEPT(p = std::move(ss)); } } diff --git a/libcxx/test/std/input.output/filesystems/fs.enum/enum.copy_options.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.enum/enum.copy_options.pass.cpp index 56322575fef2..4ef28ee01d8d 100644 --- a/libcxx/test/std/input.output/filesystems/fs.enum/enum.copy_options.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.enum/enum.copy_options.pass.cpp @@ -30,7 +30,7 @@ int main(int, char**) { typedef std::underlying_type::type UT; static_assert(!std::is_convertible::value, ""); - LIBCPP_ONLY(static_assert(std::is_same::value, "")); // Implementation detail + LIBCPP_STATIC_ASSERT(std::is_same::value, ""); // Implementation detail typedef check_bitmask_type BitmaskTester; assert(BitmaskTester::check()); diff --git a/libcxx/test/std/input.output/filesystems/fs.enum/enum.directory_options.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.enum/enum.directory_options.pass.cpp index 2597457ed747..4480b1e4e335 100644 --- a/libcxx/test/std/input.output/filesystems/fs.enum/enum.directory_options.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.enum/enum.directory_options.pass.cpp @@ -29,7 +29,7 @@ int main(int, char**) { // Check that E is a scoped enum by checking for conversions. typedef std::underlying_type::type UT; static_assert(!std::is_convertible::value, ""); - LIBCPP_ONLY(static_assert(std::is_same::value, "")); + LIBCPP_STATIC_ASSERT(std::is_same::value, ""); typedef check_bitmask_type BitmaskTester; assert(BitmaskTester::check()); diff --git a/libcxx/test/std/input.output/filesystems/fs.enum/enum.file_type.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.enum/enum.file_type.pass.cpp index 63db8892b092..6062935126dd 100644 --- a/libcxx/test/std/input.output/filesystems/fs.enum/enum.file_type.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.enum/enum.file_type.pass.cpp @@ -29,7 +29,7 @@ int main(int, char**) { typedef std::underlying_type::type UT; static_assert(!std::is_convertible::value, ""); - LIBCPP_ONLY(static_assert(std::is_same::value, "")); // Implementation detail + LIBCPP_STATIC_ASSERT(std::is_same::value, ""); // Implementation detail // The standard doesn't specify the numeric values of the enum. LIBCPP_STATIC_ASSERT( diff --git a/libcxx/test/std/input.output/filesystems/fs.enum/enum.path.format.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.enum/enum.path.format.pass.cpp index 7f3022a3ce9b..ad0cdb092def 100644 --- a/libcxx/test/std/input.output/filesystems/fs.enum/enum.path.format.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.enum/enum.path.format.pass.cpp @@ -25,9 +25,7 @@ int main(int, char**) { typedef fs::path::format E; static_assert(std::is_enum::value, ""); - typedef std::underlying_type::type UT; - - LIBCPP_ONLY(static_assert(std::is_same::value, "")); // Implementation detail + LIBCPP_STATIC_ASSERT(std::is_same::type, unsigned char>::value, ""); // Implementation detail static_assert( E::auto_format != E::native_format && diff --git a/libcxx/test/std/input.output/filesystems/fs.enum/enum.perm_options.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.enum/enum.perm_options.pass.cpp index afc64f6f00b4..0bdae4870238 100644 --- a/libcxx/test/std/input.output/filesystems/fs.enum/enum.perm_options.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.enum/enum.perm_options.pass.cpp @@ -32,7 +32,7 @@ int main(int, char**) { typedef std::underlying_type::type UT; static_assert(!std::is_convertible::value, ""); - LIBCPP_ONLY(static_assert(std::is_same::value, "")); // Implementation detail + LIBCPP_STATIC_ASSERT(std::is_same::value, ""); // Implementation detail typedef check_bitmask_type BitmaskTester; assert(BitmaskTester::check()); diff --git a/libcxx/test/std/input.output/filesystems/fs.enum/enum.perms.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.enum/enum.perms.pass.cpp index 27aa5e29de20..d1893847f01b 100644 --- a/libcxx/test/std/input.output/filesystems/fs.enum/enum.perms.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.enum/enum.perms.pass.cpp @@ -30,7 +30,7 @@ int main(int, char**) { typedef std::underlying_type::type UT; static_assert(!std::is_convertible::value, ""); - LIBCPP_ONLY(static_assert(std::is_same::value, "")); // Implementation detail + LIBCPP_STATIC_ASSERT(std::is_same::value, ""); // Implementation detail typedef check_bitmask_type BitmaskTester; assert(BitmaskTester::check()); diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.absolute/absolute.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.absolute/absolute.pass.cpp index 68fc09157c4b..7a60d1ab29f4 100644 --- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.absolute/absolute.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.absolute/absolute.pass.cpp @@ -51,7 +51,7 @@ static void basic_test() assert(!ec); assert(ret.is_absolute()); assert(PathEqIgnoreSep(ret, TC.expect)); - LIBCPP_ONLY(assert(PathEq(ret, TC.expect))); + LIBCPP_ASSERT(PathEq(ret, TC.expect)); } } diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp index 6e947a355a54..0098fe8ee698 100644 --- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.canonical/canonical.pass.cpp @@ -104,7 +104,7 @@ static void test_exception_contains_paths() } catch (filesystem_error const& err) { assert(err.path1() == p); // libc++ provides the current path as the second path in the exception - LIBCPP_ONLY(assert(err.path2() == current_path())); + LIBCPP_ASSERT(err.path2() == current_path()); } fs::current_path(static_env.Dir); try { @@ -112,7 +112,7 @@ static void test_exception_contains_paths() assert(false); } catch (filesystem_error const& err) { assert(err.path1() == p); - LIBCPP_ONLY(assert(err.path2() == static_env.Dir)); + LIBCPP_ASSERT(err.path2() == static_env.Dir); } #endif } diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp index 0f5a7692bb70..f009befa49c3 100644 --- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.permissions/permissions.pass.cpp @@ -40,7 +40,7 @@ static void test_signatures() ASSERT_NOT_NOEXCEPT(fs::permissions(p, pr)); ASSERT_NOT_NOEXCEPT(fs::permissions(p, pr, opts)); ASSERT_NOEXCEPT(fs::permissions(p, pr, ec)); - LIBCPP_ONLY(ASSERT_NOT_NOEXCEPT(fs::permissions(p, pr, opts, ec))); + LIBCPP_ASSERT_NOT_NOEXCEPT(fs::permissions(p, pr, opts, ec)); } static void test_error_reporting() diff --git a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp index c0a98a62e6f9..aed87b73121d 100644 --- a/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp +++ b/libcxx/test/std/input.output/filesystems/fs.op.funcs/fs.op.temp_dir_path/temp_directory_path.pass.cpp @@ -95,7 +95,7 @@ static void basic_tests() PutEnv(TC.name, dne); ec = GetTestEC(); ret = temp_directory_path(ec); - LIBCPP_ONLY(assert(ErrorIs(ec, expect_errc))); + LIBCPP_ASSERT(ErrorIs(ec, expect_errc)); assert(ec != GetTestEC()); assert(ec); assert(ret == ""); @@ -104,7 +104,7 @@ static void basic_tests() PutEnv(TC.name, file); ec = GetTestEC(); ret = temp_directory_path(ec); - LIBCPP_ONLY(assert(ErrorIs(ec, expect_errc))); + LIBCPP_ASSERT(ErrorIs(ec, expect_errc)); assert(ec != GetTestEC()); assert(ec); assert(ret == ""); diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp index c9297318cd5d..f502616b677b 100644 --- a/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp +++ b/libcxx/test/std/input.output/iostream.format/print.fun/no_file_description.pass.cpp @@ -69,7 +69,8 @@ static void test_vprint_unicode() { FILE* file = fmemopen(buffer.data(), buffer.size(), "wb"); assert(file); - std::vprint_unicode(file, "hello world{}", std::make_format_args('!')); + char c = '!'; + std::vprint_unicode(file, "hello world{}", std::make_format_args(c)); long pos = std::ftell(file); std::fclose(file); @@ -83,7 +84,8 @@ static void test_vprint_nonunicode() { FILE* file = fmemopen(buffer.data(), buffer.size(), "wb"); assert(file); - std::vprint_nonunicode(file, "hello world{}", std::make_format_args('!')); + char c = '!'; + std::vprint_nonunicode(file, "hello world{}", std::make_format_args(c)); long pos = std::ftell(file); std::fclose(file); diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/vprint_nonunicode.sh.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/vprint_nonunicode.sh.cpp index 63e1d2ad82b7..c1a690f559b1 100644 --- a/libcxx/test/std/input.output/iostream.format/print.fun/vprint_nonunicode.sh.cpp +++ b/libcxx/test/std/input.output/iostream.format/print.fun/vprint_nonunicode.sh.cpp @@ -35,8 +35,12 @@ int main(int, char**) { // The data is passed as-is so it does not depend on the encoding of the input. - std::vprint_nonunicode("{} {} ", std::make_format_args(1234, "一二三四")); - std::vprint_nonunicode("{} {}", std::make_format_args(true, nullptr)); + int i = 1234; + const char* s = "一二三四"; + bool b = true; + nullptr_t p = nullptr; + std::vprint_nonunicode("{} {} ", std::make_format_args(i, s)); + std::vprint_nonunicode("{} {}", std::make_format_args(b, p)); return 0; } diff --git a/libcxx/test/std/input.output/iostream.format/print.fun/vprint_unicode.sh.cpp b/libcxx/test/std/input.output/iostream.format/print.fun/vprint_unicode.sh.cpp index a9bcc33d2e01..198e71b55d9b 100644 --- a/libcxx/test/std/input.output/iostream.format/print.fun/vprint_unicode.sh.cpp +++ b/libcxx/test/std/input.output/iostream.format/print.fun/vprint_unicode.sh.cpp @@ -35,8 +35,12 @@ int main(int, char**) { // The data is passed as-is so it does not depend on the encoding of the input. - std::vprint_unicode("{} {} ", std::make_format_args(1234, "一二三四")); - std::vprint_unicode("{} {}", std::make_format_args(true, nullptr)); + int i = 1234; + const char* s = "一二三四"; + bool b = true; + nullptr_t p = nullptr; + std::vprint_unicode("{} {} ", std::make_format_args(i, s)); + std::vprint_unicode("{} {}", std::make_format_args(b, p)); return 0; } diff --git a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp index ad6cc192e2f4..ce6a7f37801e 100644 --- a/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp +++ b/libcxx/test/std/input.output/iostream.objects/narrow.stream.objects/cin.sh.cpp @@ -8,7 +8,11 @@ // TODO: Investigate // UNSUPPORTED: LIBCXX-AIX-FIXME -// XFAIL: LIBCXX-PICOLIBC-FIXME + +// QEMU does not detect EOF, when reading from stdin +// "echo -n" suppresses any characters after the output and so the test hangs. +// https://gitlab.com/qemu-project/qemu/-/issues/1963 +// UNSUPPORTED: LIBCXX-PICOLIBC-FIXME // This test hangs on Android devices that lack shell_v2, which was added in // Android N (API 24). diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/member_swap_noexcept.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/member_swap_noexcept.pass.cpp index 0a0128e44658..63e2293c530b 100644 --- a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/member_swap_noexcept.pass.cpp +++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/member_swap_noexcept.pass.cpp @@ -84,7 +84,7 @@ static void test() { { std::basic_stringbuf, test_alloc_not_empty> buf1; std::basic_stringbuf, test_alloc_not_empty> buf; - static_assert(!noexcept(buf.swap(buf1))); + LIBCPP_STATIC_ASSERT(!noexcept(buf.swap(buf1))); } { std::basic_stringbuf, test_alloc_propagate_on_container_swap_not_empty> buf1; diff --git a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/nonmember_swap_noexcept.pass.cpp b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/nonmember_swap_noexcept.pass.cpp index 4f41e3a4d716..a3706116f409 100644 --- a/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/nonmember_swap_noexcept.pass.cpp +++ b/libcxx/test/std/input.output/string.streams/stringbuf/stringbuf.assign/nonmember_swap_noexcept.pass.cpp @@ -83,7 +83,7 @@ static void test() { { std::basic_stringbuf, test_alloc_not_empty> buf1; std::basic_stringbuf, test_alloc_not_empty> buf; - static_assert(!noexcept(swap(buf, buf1))); + LIBCPP_STATIC_ASSERT(!noexcept(swap(buf, buf1))); } { std::basic_stringbuf, test_alloc_propagate_on_container_swap_not_empty> buf1; diff --git a/libcxx/test/std/input.output/syncstream/syncbuf/syncstream.syncbuf.assign/swap.pass.cpp b/libcxx/test/std/input.output/syncstream/syncbuf/syncstream.syncbuf.assign/swap.pass.cpp index ba007da5a054..a236bf4752a0 100644 --- a/libcxx/test/std/input.output/syncstream/syncbuf/syncstream.syncbuf.assign/swap.pass.cpp +++ b/libcxx/test/std/input.output/syncstream/syncbuf/syncstream.syncbuf.assign/swap.pass.cpp @@ -75,7 +75,8 @@ static void test_short_write_after_swap() { sync_buf2.sputn(expected.data(), expected.size()); sync_buf1.swap(sync_buf2); - expected.push_back(sync_buf1.sputc(CharT('B'))); + sync_buf1.sputc(CharT('B')); + expected.push_back(CharT('B')); sync_buf2.sputc(CharT('Z')); assert(sstr1.str().empty()); diff --git a/libcxx/test/std/iterators/iterator.requirements/iterator.cust/iterator.cust.move/iter_move.pass.cpp b/libcxx/test/std/iterators/iterator.requirements/iterator.cust/iterator.cust.move/iter_move.pass.cpp index 566638263e88..9f293ff483cd 100644 --- a/libcxx/test/std/iterators/iterator.requirements/iterator.cust/iterator.cust.move/iter_move.pass.cpp +++ b/libcxx/test/std/iterators/iterator.requirements/iterator.cust/iterator.cust.move/iter_move.pass.cpp @@ -158,15 +158,15 @@ constexpr bool test() { auto unscoped = check_unqualified_lookup::unscoped_enum::a; assert(std::ranges::iter_move(unscoped) == check_unqualified_lookup::unscoped_enum::a); - assert(!noexcept(std::ranges::iter_move(unscoped))); + static_assert(!noexcept(std::ranges::iter_move(unscoped))); auto scoped = check_unqualified_lookup::scoped_enum::a; assert(std::ranges::iter_move(scoped) == nullptr); - assert(noexcept(std::ranges::iter_move(scoped))); + static_assert(noexcept(std::ranges::iter_move(scoped))); auto some_union = check_unqualified_lookup::some_union{0}; assert(std::ranges::iter_move(some_union) == 0); - assert(!noexcept(std::ranges::iter_move(some_union))); + static_assert(!noexcept(std::ranges::iter_move(some_union))); // Check noexcept-correctness static_assert(noexcept(std::ranges::iter_move(std::declval>()))); diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.indirect.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.indirect.pass.cpp index 172b6cc2f294..f6f586a2b547 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.indirect.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.indirect.pass.cpp @@ -29,7 +29,9 @@ TEST_WORKAROUND_BUG_109234844_WEAK void* operator new(std::size_t s) TEST_THROW_SPEC(std::bad_alloc) { ++new_called; void* ret = std::malloc(s); - if (!ret) std::abort(); // placate MSVC's unchecked malloc warning + if (!ret) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } return ret; } diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.pass.cpp index e352c00b4d0a..29e739d8515f 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size.replace.pass.cpp @@ -27,7 +27,9 @@ int delete_called = 0; void* operator new[](std::size_t s) TEST_THROW_SPEC(std::bad_alloc) { ++new_called; void* ret = std::malloc(s); - if (!ret) std::abort(); // placate MSVC's unchecked malloc warning + if (!ret) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } return ret; } diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_nothrow.replace.indirect.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_nothrow.replace.indirect.pass.cpp index 8ad0292dcb5c..b26eec0324af 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_nothrow.replace.indirect.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.array/new.size_nothrow.replace.indirect.pass.cpp @@ -33,7 +33,9 @@ TEST_WORKAROUND_BUG_109234844_WEAK void* operator new[](std::size_t s) TEST_THROW_SPEC(std::bad_alloc) { ++new_called; void* ret = std::malloc(s); - if (!ret) std::abort(); // placate MSVC's unchecked malloc warning + if (!ret) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } return ret; } diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.replace.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.replace.pass.cpp index a03313e5872e..ab1cf5ea4644 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.replace.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size.replace.pass.cpp @@ -25,7 +25,9 @@ int delete_called = 0; void* operator new(std::size_t s) TEST_THROW_SPEC(std::bad_alloc) { ++new_called; void* ret = std::malloc(s); - if (!ret) std::abort(); // placate MSVC's unchecked malloc warning + if (!ret) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } return ret; } diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align_nothrow.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align_nothrow.pass.cpp index 4e5d36cd7c6d..1c575729678d 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align_nothrow.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_align_nothrow.pass.cpp @@ -60,8 +60,9 @@ int main(int, char**) { assert(reinterpret_cast(x) % alignof(TrackLifetimeOverAligned) == 0); assert(info.address_constructed == x); + const auto old_x = x; delete x; - assert(info.address_destroyed == x); + assert(info.address_destroyed == old_x); } return 0; diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.pass.cpp index 398de0068aba..56ae8df43f66 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.pass.cpp @@ -50,8 +50,9 @@ int main(int, char**) { assert(x != nullptr); assert(info.address_constructed == x); + const auto old_x = x; delete x; - assert(info.address_destroyed == x); + assert(info.address_destroyed == old_x); } return 0; diff --git a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.replace.indirect.pass.cpp b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.replace.indirect.pass.cpp index 2ae0dfa4f1ab..35a601339ddd 100644 --- a/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.replace.indirect.pass.cpp +++ b/libcxx/test/std/language.support/support.dynamic/new.delete/new.delete.single/new.size_nothrow.replace.indirect.pass.cpp @@ -28,7 +28,9 @@ TEST_WORKAROUND_BUG_109234844_WEAK void* operator new(std::size_t s) TEST_THROW_SPEC(std::bad_alloc) { ++new_called; void* ret = std::malloc(s); - if (!ret) std::abort(); // placate MSVC's unchecked malloc warning + if (!ret) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } return ret; } diff --git a/libcxx/test/std/modules/std.compat.pass.cpp b/libcxx/test/std/modules/std.compat.pass.cpp new file mode 100644 index 000000000000..a33ed3b6b645 --- /dev/null +++ b/libcxx/test/std/modules/std.compat.pass.cpp @@ -0,0 +1,18 @@ +//===----------------------------------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +// UNSUPPORTED: c++03, c++11, c++14, c++17 + +// UNSUPPORTED: libcpp-has-no-std-modules +// UNSUPPORTED: clang-modules-build + +// A minimal test to validate import works. + +import std.compat; + +int main(int, char**) { return !(::strlen("Hello modular world") == 19); } diff --git a/libcxx/test/std/ranges/range.adaptors/range.elements/general.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.elements/general.pass.cpp index 78792ae54bdb..d5318ced73dc 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.elements/general.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.elements/general.pass.cpp @@ -70,7 +70,7 @@ int main(int, char**) { // tuple { - std::tuple tps[] = {{1}, {2}, {3}}; + std::tuple tps[] = {{short{1}}, {short{2}}, {short{3}}}; auto ev = tps | std::views::elements<0>; auto expected = {1, 2, 3}; assert(std::ranges::equal(ev, expected)); diff --git a/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/base.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/base.pass.cpp index 3729c8e54311..79c4e1f41837 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/base.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/base.pass.cpp @@ -33,7 +33,7 @@ using ElementsIter = std::ranges::iterator_t); static_assert(IsBaseNoexcept); static_assert(IsBaseNoexcept); -static_assert(!IsBaseNoexcept); +LIBCPP_STATIC_ASSERT(!IsBaseNoexcept); constexpr bool test() { std::tuple t{5}; diff --git a/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/deref.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/deref.pass.cpp index d87a3e533920..f88091f42699 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/deref.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/deref.pass.cpp @@ -50,7 +50,7 @@ constexpr void testValue(T t) { constexpr bool test() { // test tuple { - std::tuple ts[] = {{1, 2, 3}, {4, 5, 6}}; + std::tuple ts[] = {{1, short{2}, 3}, {4, short{5}, 6}}; testReference<0>(ts); testReference<1>(ts); testReference<2>(ts); @@ -61,7 +61,7 @@ constexpr bool test() { // test pair { - std::pair ps[] = {{1, 2}, {4, 5}}; + std::pair ps[] = {{1, short{2}}, {4, short{5}}}; testReference<0>(ps); testReference<1>(ps); testValue<0>(ps[0]); diff --git a/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/member_types.compile.pass.cpp b/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/member_types.compile.pass.cpp index 9a76c2fcb70c..70d49c1304b5 100644 --- a/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/member_types.compile.pass.cpp +++ b/libcxx/test/std/ranges/range.adaptors/range.elements/iterator/member_types.compile.pass.cpp @@ -64,9 +64,7 @@ static_assert(std::same_as*>>::iterator_category, // std::random_access_iterator_tag>); -using Generator = decltype(std::views::iota(0, 1) | std::views::transform([](int) { - return std::pair{1, 1}; - })); +using Generator = decltype(std::views::iota(0, 1) | std::views::transform([](int) { return std::pair{1, short{1}}; })); static_assert(std::ranges::random_access_range); static_assert(std::same_as::iterator_category, // diff --git a/libcxx/test/std/ranges/range.utility/range.utility.conv/container.h b/libcxx/test/std/ranges/range.utility/range.utility.conv/container.h index fafccacd456d..ca89e3757aff 100644 --- a/libcxx/test/std/ranges/range.utility/range.utility.conv/container.h +++ b/libcxx/test/std/ranges/range.utility/range.utility.conv/container.h @@ -38,7 +38,7 @@ struct Container { constexpr explicit Container(std::ranges::input_range auto&& in) requires(Rank >= CtrChoice::DirectCtr) - : ctr_choice(CtrChoice::DirectCtr), size_(std::ranges::size(in)) { + : ctr_choice(CtrChoice::DirectCtr), size_(static_cast(std::ranges::size(in))) { std::ranges::copy(in, begin()); } @@ -54,7 +54,7 @@ struct Container { constexpr Container(std::from_range_t, std::ranges::input_range auto&& in) requires(Rank >= CtrChoice::FromRangeT) - : ctr_choice(CtrChoice::FromRangeT), size_(std::ranges::size(in)) { + : ctr_choice(CtrChoice::FromRangeT), size_(static_cast(std::ranges::size(in))) { std::ranges::copy(in, begin()); } @@ -70,7 +70,7 @@ struct Container { template constexpr Container(Iter b, Iter e) requires(Rank >= CtrChoice::BeginEndPair) - : ctr_choice(CtrChoice::BeginEndPair), size_(e - b) { + : ctr_choice(CtrChoice::BeginEndPair), size_(static_cast(e - b)) { std::ranges::copy(b, e, begin()); } diff --git a/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/F.pass.cpp b/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/F.pass.cpp index fc3d5c122274..1c520652ba22 100644 --- a/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/F.pass.cpp +++ b/libcxx/test/std/thread/thread.threads/thread.thread.class/thread.thread.constr/F.pass.cpp @@ -36,7 +36,9 @@ void* operator new(std::size_t s) TEST_THROW_SPEC(std::bad_alloc) } while (!throw_one.compare_exchange_weak(expected, expected - 1)); ++outstanding_new; void* ret = std::malloc(s); - if (!ret) std::abort(); // placate MSVC's unchecked malloc warning + if (!ret) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } return ret; } diff --git a/libcxx/test/std/time/time.syn/formatter_tests.h b/libcxx/test/std/time/time.syn/formatter_tests.h index 42c176b3b47e..1b343b5c8711 100644 --- a/libcxx/test/std/time/time.syn/formatter_tests.h +++ b/libcxx/test/std/time/time.syn/formatter_tests.h @@ -8,6 +8,8 @@ #ifndef TEST_STD_TIME_TIME_SYN_FORMATTER_TESTS_H #define TEST_STD_TIME_TIME_SYN_FORMATTER_TESTS_H +#include "assert_macros.h" +#include "concat_macros.h" #include "make_string.h" #include "string_literal.h" #include "test_format_string.h" @@ -34,11 +36,9 @@ using format_context = std::format_context; template void check(std::basic_string_view expected, test_format_string fmt, Args&&... args) { std::basic_string out = std::format(fmt, std::forward(args)...); - if constexpr (std::same_as) - if (out != expected) - std::cerr << "\nFormat string " << fmt.get() << "\nExpected output " << expected << "\nActual output " << out - << '\n'; - assert(out == expected); + TEST_REQUIRE(out == expected, + TEST_WRITE_CONCATENATED( + "\nFormat string ", fmt.get(), "\nExpected output ", expected, "\nActual output ", out, '\n')); } template @@ -47,38 +47,24 @@ void check(const std::locale& loc, test_format_string fmt, Args&&... args) { std::basic_string out = std::format(loc, fmt, std::forward(args)...); - if constexpr (std::same_as) - if (out != expected) - std::cerr << "\nFormat string " << fmt.get() << "\nExpected output " << expected << "\nActual output " << out - << '\n'; - assert(out == expected); + TEST_REQUIRE(out == expected, + TEST_WRITE_CONCATENATED( + "\nFormat string ", fmt.get(), "\nExpected output ", expected, "\nActual output ", out, '\n')); } template void check_exception([[maybe_unused]] std::string_view what, [[maybe_unused]] std::basic_string_view fmt, [[maybe_unused]] const Args&... args) { -#ifndef TEST_HAS_NO_EXCEPTIONS - try { - TEST_IGNORE_NODISCARD std::vformat(fmt, std::make_format_args>(args...)); - if constexpr (std::same_as) - std::cerr << "\nFormat string " << fmt << "\nDidn't throw an exception.\n"; - assert(false); - } catch (const std::format_error& e) { -# if defined(_LIBCPP_VERSION) - if constexpr (std::same_as) - if (e.what() != what) - std::cerr << "\nFormat string " << fmt << "\nExpected exception " << what << "\nActual exception " - << e.what() << '\n'; - assert(e.what() == what); -# else - (void)what; - (void)e; -# endif - return; - } - assert(false); -#endif + TEST_VALIDATE_EXCEPTION( + std::format_error, + [&]([[maybe_unused]] const std::format_error& e) { + TEST_LIBCPP_REQUIRE( + e.what() == what, + TEST_WRITE_CONCATENATED( + "\nFormat string ", fmt, "\nExpected exception ", what, "\nActual exception ", e.what(), '\n')); + }, + TEST_IGNORE_NODISCARD std::vformat(fmt, std::make_format_args>(args...))); } template diff --git a/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.pass.cpp b/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.pass.cpp index 72184bbd3920..62fd0f25ae3a 100644 --- a/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.pass.cpp +++ b/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.pass.cpp @@ -10,7 +10,7 @@ // // template -// format-arg-store make_format_args(const Args&... args); +// format-arg-store make_format_args(Args&... args); #include #include @@ -20,8 +20,19 @@ #include "test_basic_format_arg.h" #include "test_macros.h" +template +concept can_make_format_args = requires(Args&&... args) { std::make_format_args(std::forward(args)...); }; + +static_assert(can_make_format_args); +static_assert(!can_make_format_args); +static_assert(!can_make_format_args); + int main(int, char**) { - [[maybe_unused]] auto store = std::make_format_args(42, nullptr, false, 'x'); + int i = 1; + char c = 'c'; + nullptr_t p = nullptr; + bool b = false; + [[maybe_unused]] auto store = std::make_format_args(i, p, b, c); LIBCPP_STATIC_ASSERT( std::same_as>); diff --git a/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.sh.cpp b/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.sh.cpp index 95a94b0bff5d..2d5ee8349b74 100644 --- a/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.sh.cpp +++ b/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_format_args.sh.cpp @@ -24,6 +24,7 @@ #include "test_macros.h" void test() { + char c = 'c'; TEST_IGNORE_NODISCARD - std::make_format_args>, wchar_t>>('c'); + std::make_format_args>, wchar_t>>(c); } diff --git a/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_wformat_args.pass.cpp b/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_wformat_args.pass.cpp index 22c6f031efc6..73c4395a4a63 100644 --- a/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_wformat_args.pass.cpp +++ b/libcxx/test/std/utilities/format/format.arguments/format.arg.store/make_wformat_args.pass.cpp @@ -12,7 +12,7 @@ // template // format-arg-store -// make_wformat_args(const Args&... args); +// make_wformat_args(Args&... args); #include #include @@ -20,8 +20,19 @@ #include "test_basic_format_arg.h" #include "test_macros.h" +template +concept can_make_wformat_args = requires(Args&&... args) { std::make_wformat_args(std::forward(args)...); }; + +static_assert(can_make_wformat_args); +static_assert(!can_make_wformat_args); +static_assert(!can_make_wformat_args); + int main(int, char**) { - [[maybe_unused]] auto store = std::make_wformat_args(42, nullptr, false, 'x'); + int i = 1; + char c = 'c'; + nullptr_t p = nullptr; + bool b = false; + [[maybe_unused]] auto store = std::make_wformat_args(i, p, b, c); LIBCPP_STATIC_ASSERT( std::same_as>); diff --git a/libcxx/test/std/utilities/format/format.arguments/format.args/ctad.compile.pass.cpp b/libcxx/test/std/utilities/format/format.arguments/format.args/ctad.compile.pass.cpp index 7cadd4e76c74..b87b5c774ef7 100644 --- a/libcxx/test/std/utilities/format/format.arguments/format.args/ctad.compile.pass.cpp +++ b/libcxx/test/std/utilities/format/format.arguments/format.args/ctad.compile.pass.cpp @@ -18,12 +18,13 @@ #include "test_macros.h" void test() { + int i = 1; // Note the Standard way to create a format-arg-store is by using make_format_args. - static_assert(std::same_as>); #ifndef TEST_HAS_NO_WIDE_CHARACTERS - static_assert(std::same_as>); #endif diff --git a/libcxx/test/std/utilities/format/format.arguments/format.args/ctor.pass.cpp b/libcxx/test/std/utilities/format/format.arguments/format.args/ctor.pass.cpp index c2d2d19d978b..c0575c545bde 100644 --- a/libcxx/test/std/utilities/format/format.arguments/format.args/ctor.pass.cpp +++ b/libcxx/test/std/utilities/format/format.arguments/format.args/ctor.pass.cpp @@ -20,6 +20,9 @@ template void test() { + int i = 1; + char c = 'c'; + nullptr_t p = nullptr; using Context = std::basic_format_context; { ASSERT_NOEXCEPT(std::basic_format_args{}); @@ -28,14 +31,14 @@ void test() { assert(!format_args.get(0)); } { - auto store = std::make_format_args(1); + auto store = std::make_format_args(i); ASSERT_NOEXCEPT(std::basic_format_args{store}); std::basic_format_args format_args{store}; assert(format_args.get(0)); assert(!format_args.get(1)); } { - auto store = std::make_format_args(1, 'c'); + auto store = std::make_format_args(i, c); ASSERT_NOEXCEPT(std::basic_format_args{store}); std::basic_format_args format_args{store}; assert(format_args.get(0)); @@ -43,7 +46,7 @@ void test() { assert(!format_args.get(2)); } { - auto store = std::make_format_args(1, 'c', nullptr); + auto store = std::make_format_args(i, c, p); ASSERT_NOEXCEPT(std::basic_format_args{store}); std::basic_format_args format_args{store}; assert(format_args.get(0)); diff --git a/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/arg.pass.cpp b/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/arg.pass.cpp index 153f8bbaa4b3..824813d33a51 100644 --- a/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/arg.pass.cpp +++ b/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/arg.pass.cpp @@ -23,9 +23,11 @@ template void test() { + bool b = true; + CharT c = CharT('a'); + int a = 42; std::basic_string string = MAKE_STRING(CharT, "string"); - auto store = std::make_format_args>( - true, CharT('a'), 42, string); + auto store = std::make_format_args>(b, c, a, string); std::basic_format_args args = store; std::basic_string output; diff --git a/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/ctor.pass.cpp b/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/ctor.pass.cpp index 4384f7b0fe68..40720105060f 100644 --- a/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/ctor.pass.cpp +++ b/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/ctor.pass.cpp @@ -36,11 +36,14 @@ template void test() { + int a = 42; + bool b = true; + CharT c = CharT('a'); std::basic_string string = MAKE_STRING(CharT, "string"); // The type of the object is an exposition only type. The temporary is needed // to extend the lifetime of the object since args stores a pointer to the // data in this object. - auto format_arg_store = std::make_format_args>(true, CharT('a'), 42, string); + auto format_arg_store = std::make_format_args>(b, c, a, string); std::basic_format_args args = format_arg_store; { diff --git a/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/locale.pass.cpp b/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/locale.pass.cpp index 5533fe1b9f28..14bdc1426098 100644 --- a/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/locale.pass.cpp +++ b/libcxx/test/std/utilities/format/format.formatter/format.context/format.context/locale.pass.cpp @@ -34,7 +34,10 @@ void test() { // The type of the object is an exposition only type. The temporary is needed // to extend the lifetime of the object since args stores a pointer to the // data in this object. - auto format_arg_store = std::make_format_args>(true, CharT('a'), 42, string); + int a = 42; + bool b = true; + CharT c = CharT('a'); + auto format_arg_store = std::make_format_args>(b, c, a, string); std::basic_format_args args = format_arg_store; { diff --git a/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.string.pass.cpp b/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.string.pass.cpp index 292de0cdd845..e99deb2db553 100644 --- a/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.string.pass.cpp +++ b/libcxx/test/std/utilities/format/format.formatter/format.formatter.spec/formatter.string.pass.cpp @@ -53,8 +53,7 @@ void test(StringT expected, StringViewT fmt, StringT a, std::size_t offset) { using FormatCtxT = std::basic_format_context; ArgumentT arg = a; - FormatCtxT format_ctx = test_format_context_create( - out, std::make_format_args(std::forward(arg))); + FormatCtxT format_ctx = test_format_context_create(out, std::make_format_args(arg)); formatter.format(arg, format_ctx); assert(result == expected); } diff --git a/libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp b/libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp index 5c6f86f742ba..23b26722d7be 100644 --- a/libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp +++ b/libcxx/test/std/utilities/format/format.functions/escaped_output.unicode.pass.cpp @@ -515,7 +515,7 @@ static void test_ill_formed_utf8() { } #ifndef TEST_HAS_NO_WIDE_CHARACTERS -# ifdef _LIBCPP_SHORT_WCHAR +# ifdef TEST_SHORT_WCHAR static void test_ill_formed_utf16() { using namespace std::literals; @@ -543,7 +543,7 @@ static void test_ill_formed_utf16() { L"\xdbff" "a"); } -# else // _LIBCPP_SHORT_WCHAR +# else // TEST_SHORT_WCHAR static void test_ill_formed_utf32() { using namespace std::literals; @@ -552,7 +552,7 @@ static void test_ill_formed_utf32() { test_format(LR"("\x{ffffffff}")"sv, L"{:?}", L"\xffffffff"); // largest encoded code point } -# endif // _LIBCPP_SHORT_WCHAR +# endif // TEST_SHORT_WCHAR #endif // TEST_HAS_NO_WIDE_CHARACTERS int main(int, char**) { @@ -563,11 +563,11 @@ int main(int, char**) { test_ill_formed_utf8(); #ifndef TEST_HAS_NO_WIDE_CHARACTERS -# ifdef _LIBCPP_SHORT_WCHAR +# ifdef TEST_SHORT_WCHAR test_ill_formed_utf16(); -# else // _LIBCPP_SHORT_WCHAR +# else // TEST_SHORT_WCHAR test_ill_formed_utf32(); -# endif // _LIBCPP_SHORT_WCHAR +# endif // TEST_SHORT_WCHAR #endif // TEST_HAS_NO_WIDE_CHARACTERS return 0; diff --git a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.const/type_conv_ctor.pass.cpp b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.const/type_conv_ctor.pass.cpp index 2759b921fabe..48c62ae7d45b 100644 --- a/libcxx/test/std/utilities/function.objects/refwrap/refwrap.const/type_conv_ctor.pass.cpp +++ b/libcxx/test/std/utilities/function.objects/refwrap/refwrap.const/type_conv_ctor.pass.cpp @@ -37,8 +37,9 @@ struct convertible_from_int { void meow(std::reference_wrapper) {} void meow(convertible_from_int) {} -int main(int, char**) -{ +std::reference_wrapper purr(); + +int main(int, char**) { { convertible_to_int_ref t; std::reference_wrapper r(t); @@ -54,21 +55,18 @@ int main(int, char**) ASSERT_NOEXCEPT(Ref(nothrow_convertible())); ASSERT_NOT_NOEXCEPT(Ref(nothrow_convertible())); } - { - meow(0); - } - { - extern std::reference_wrapper purr(); - ASSERT_SAME_TYPE(decltype(true ? purr() : 0), int); - } + meow(0); + ASSERT_SAME_TYPE(decltype(true ? purr() : 0), int); #if TEST_STD_VER > 14 { int i = 0; std::reference_wrapper ri(i); - static_assert((std::is_same>::value), "" ); + static_assert((std::is_same>::value), ""); + } + { const int j = 0; std::reference_wrapper rj(j); - static_assert((std::is_same>::value), "" ); + static_assert((std::is_same>::value), ""); } #endif diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_for_overwrite.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_for_overwrite.pass.cpp index 27ff3cd56374..e6e063304453 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_for_overwrite.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/allocate_shared_for_overwrite.pass.cpp @@ -156,7 +156,7 @@ void testAllocatorOperationsCalled() { template struct AllocatorWithPattern { - constexpr static char pattern = 0xDE; + constexpr static char pattern = static_cast(0xDE); using value_type = T; diff --git a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared_for_overwrite.pass.cpp b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared_for_overwrite.pass.cpp index 21e1786f0158..96363060a7be 100644 --- a/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared_for_overwrite.pass.cpp +++ b/libcxx/test/std/utilities/memory/util.smartptr/util.smartptr.shared/util.smartptr.shared.create/make_shared_for_overwrite.pass.cpp @@ -56,10 +56,13 @@ static_assert(!HasMakeSharedForOverwrite); static_assert(!HasMakeSharedForOverwrite); static_assert(!HasMakeSharedForOverwrite); -constexpr char pattern = 0xDE; +constexpr char pattern = static_cast(0xDE); void* operator new(std::size_t count) { void* ptr = std::malloc(count); + if (!ptr) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } for (std::size_t i = 0; i < count; ++i) { *(reinterpret_cast(ptr) + i) = pattern; } diff --git a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.create/make_unique_for_overwrite.default_init.pass.cpp b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.create/make_unique_for_overwrite.default_init.pass.cpp index 8011a37be08e..3ac0ce962cfa 100644 --- a/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.create/make_unique_for_overwrite.default_init.pass.cpp +++ b/libcxx/test/std/utilities/smartptr/unique.ptr/unique.ptr.create/make_unique_for_overwrite.default_init.pass.cpp @@ -23,10 +23,13 @@ #include #include -constexpr char pattern = 0xDE; +constexpr char pattern = static_cast(0xDE); void* operator new(std::size_t count) { void* ptr = std::malloc(count); + if (!ptr) { + std::abort(); // placate MSVC's unchecked malloc warning (assert() won't silence it) + } for (std::size_t i = 0; i < count; ++i) { *(reinterpret_cast(ptr) + i) = pattern; } diff --git a/libcxx/test/support/allocators.h b/libcxx/test/support/allocators.h index 2b987ad87278..02436fd9c35e 100644 --- a/libcxx/test/support/allocators.h +++ b/libcxx/test/support/allocators.h @@ -209,7 +209,7 @@ public: : id_(id), copy_assigned_into_(copy_assigned_into) {} template - MaybePOCCAAllocator(const MaybePOCCAAllocator& that) + TEST_CONSTEXPR MaybePOCCAAllocator(const MaybePOCCAAllocator& that) : id_(that.id_), copy_assigned_into_(that.copy_assigned_into_) {} MaybePOCCAAllocator(const MaybePOCCAAllocator&) = default; diff --git a/libcxx/test/support/concat_macros.h b/libcxx/test/support/concat_macros.h index 8d80a8c9f69c..d7340b8faf6e 100644 --- a/libcxx/test/support/concat_macros.h +++ b/libcxx/test/support/concat_macros.h @@ -16,15 +16,154 @@ #include "test_macros.h" #ifndef TEST_HAS_NO_LOCALIZATION +# include +# include # include #endif #if TEST_STD_VER > 17 # ifndef TEST_HAS_NO_LOCALIZATION + +[[nodiscard]] constexpr bool test_is_high_surrogate(char32_t value) { return value >= 0xd800 && value <= 0xdbff; } + +[[nodiscard]] constexpr bool test_is_low_surrogate(char32_t value) { return value >= 0xdc00 && value <= 0xdfff; } + +[[nodiscard]] constexpr bool test_is_surrogate(char32_t value) { return value >= 0xd800 && value <= 0xdfff; } + +[[nodiscard]] constexpr bool test_is_code_point(char32_t value) { return value <= 0x10ffff; } + +[[nodiscard]] constexpr bool test_is_scalar_value(char32_t value) { + return test_is_code_point(value) && !test_is_surrogate(value); +} + +inline constexpr char32_t test_replacement_character = U'\ufffd'; + +template +OutIt test_transcode() = delete; + +template + requires(std::output_iterator && std::same_as, char8_t>) +OutIt test_transcode(InIt first, InIt last, OutIt out_it) { + return std::copy(first, last, out_it); +} + +template + requires std::output_iterator +void test_encode(OutIt& out_it, char16_t value) { + if (value < 0x80) + *out_it++ = static_cast(value); + else if (value < 0x800) { + *out_it++ = static_cast(0b11000000 | (value >> 6)); + *out_it++ = static_cast(0b10000000 | (value & 0b00111111)); + } else { + *out_it++ = static_cast(0b11100000 | (value >> 12)); + *out_it++ = static_cast(0b10000000 | ((value) >> 6 & 0b00111111)); + *out_it++ = static_cast(0b10000000 | (value & 0b00111111)); + } +} + +template + requires std::output_iterator +void test_encode(OutIt& out_it, char32_t value) { + if ((value & 0xffff0000) == 0) + test_encode(out_it, static_cast(value)); + else { + *out_it++ = static_cast(0b11100000 | (value >> 18)); + *out_it++ = static_cast(0b10000000 | ((value) >> 12 & 0b00111111)); + *out_it++ = static_cast(0b10000000 | ((value) >> 6 & 0b00111111)); + *out_it++ = static_cast(0b10000000 | (value & 0b00111111)); + } +} + +template + requires(std::output_iterator && + (std::same_as, char16_t> +# ifndef TEST_HAS_NO_WIDE_CHARACTERS + || (std::same_as, wchar_t> && sizeof(wchar_t) == 2) +# endif + )) +OutIt test_transcode(InIt first, InIt last, OutIt out_it) { + while (first != last) { + char32_t value = *first++; + + if (test_is_low_surrogate(value)) [[unlikely]] { + test_encode(out_it, static_cast(test_replacement_character)); + continue; + } + + if (!test_is_high_surrogate(value)) { + test_encode(out_it, static_cast(value)); + continue; + } + + if (first == last || !test_is_low_surrogate(static_cast(*first))) [[unlikely]] { + test_encode(out_it, static_cast(test_replacement_character)); + continue; + } + + value -= 0xd800; + value <<= 10; + value += static_cast(*first++) - 0xdc00; + value += 0x10000; + + if (test_is_code_point(value)) [[likely]] + test_encode(out_it, value); + else + test_encode(out_it, static_cast(test_replacement_character)); + } + + return out_it; +} + +template + requires(std::output_iterator && + (std::same_as, char32_t> +# ifndef TEST_HAS_NO_WIDE_CHARACTERS + || (std::same_as, wchar_t> && sizeof(wchar_t) == 4) +# endif + )) +OutIt test_transcode(InIt first, InIt last, OutIt out_it) { + while (first != last) { + char32_t value = *first++; + if (test_is_code_point(value)) [[likely]] + test_encode(out_it, value); + else + test_encode(out_it, static_cast(test_replacement_character)); + } + return out_it; +} + +template +concept test_streamable = requires(std::stringstream& stream, T&& value) { stream << value; }; + +template +concept test_convertable_range = (!test_streamable && requires(R&& value) { + std::basic_string_view{std::begin(value), std::end(value)}; +}); + template -concept test_char_streamable = requires(T&& value) { std::stringstream{} << std::forward(value); }; -# endif +concept test_can_concat = test_streamable || test_convertable_range; + +template +std::ostream& test_concat(std::ostream& stream, T&& value) { + return stream << value; +} + +template +std::ostream& test_concat(std::ostream& stream, T&& value) { + auto b = std::begin(value); + auto e = std::end(value); + if (b != e) { + // When T is an array it's string-literal, remove the NUL terminator. + if constexpr (std::is_array_v>) { + --e; + } + test_transcode(b, e, std::ostream_iterator{stream}); + } + return stream; +} +# endif // TEST_HAS_NO_LOCALIZATION // If possible concatenates message for the assertion function, else returns a // default message. Not being able to stream is not considered an error. For @@ -37,12 +176,12 @@ concept test_char_streamable = requires(T&& value) { std::stringstream{} << std: template std::string test_concat_message([[maybe_unused]] Args&&... args) { # ifndef TEST_HAS_NO_LOCALIZATION - if constexpr ((test_char_streamable && ...)) { + if constexpr ((test_can_concat && ...)) { std::stringstream sstr; - ((sstr << std::forward(args)), ...); + ((test_concat(sstr, std::forward(args))), ...); return sstr.str(); } else -# endif +# endif // TEST_HAS_NO_LOCALIZATION return "Message discarded since it can't be streamed to std::cerr.\n"; } diff --git a/libcxx/test/support/msvc_stdlib_force_include.h b/libcxx/test/support/msvc_stdlib_force_include.h index b5ba33bd281c..5742bbf4f557 100644 --- a/libcxx/test/support/msvc_stdlib_force_include.h +++ b/libcxx/test/support/msvc_stdlib_force_include.h @@ -100,6 +100,8 @@ const AssertionDialogAvoider assertion_dialog_avoider{}; # define TEST_STD_VER 14 #endif +#define TEST_SHORT_WCHAR + #define _LIBCPP_AVAILABILITY_THROW_BAD_ANY_CAST #ifdef __clang__ diff --git a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp index fba48d5984ea..fcb5865adf0d 100644 --- a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp +++ b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.cpp @@ -26,7 +26,10 @@ struct clang::tidy::OptionEnumMapping Mapping[] = { {libcpp::header_exportable_declarations::FileType::Header, "Header"}, {libcpp::header_exportable_declarations::FileType::ModulePartition, "ModulePartition"}, - {libcpp::header_exportable_declarations::FileType::Module, "Module"}}; + {libcpp::header_exportable_declarations::FileType::Module, "Module"}, + {libcpp::header_exportable_declarations::FileType::CHeader, "CHeader"}, + {libcpp::header_exportable_declarations::FileType::CompatModulePartition, "CompatModulePartition"}, + {libcpp::header_exportable_declarations::FileType::CompatModule, "CompatModule"}}; return ArrayRef(Mapping); } }; @@ -39,6 +42,7 @@ header_exportable_declarations::header_exportable_declarations( file_type_(Options.get("FileType", header_exportable_declarations::FileType::Unknown)), extra_header_(Options.get("ExtraHeader", "")) { switch (file_type_) { + case header_exportable_declarations::FileType::CHeader: case header_exportable_declarations::FileType::Header: if (filename_.empty()) llvm::errs() << "No filename is provided.\n"; @@ -46,10 +50,12 @@ header_exportable_declarations::header_exportable_declarations( extra_header_ = "$^"; // Use a never matching regex to silence an error message. break; case header_exportable_declarations::FileType::ModulePartition: + case header_exportable_declarations::FileType::CompatModulePartition: if (filename_.empty()) llvm::errs() << "No filename is provided.\n"; [[fallthrough]]; case header_exportable_declarations::FileType::Module: + case header_exportable_declarations::FileType::CompatModule: if (!extra_header_.empty()) llvm::errs() << "Extra headers are not allowed for modules.\n"; if (Options.get("SkipDeclarations")) @@ -70,7 +76,7 @@ header_exportable_declarations::header_exportable_declarations( auto b = s.begin(); auto e = std::find(b, s.end(), ' '); while (b != e) { - decls_.emplace(b, e); + skip_decls_.emplace(b, e); if (e == s.end()) break; b = e + 1; @@ -82,9 +88,10 @@ header_exportable_declarations::header_exportable_declarations( for (auto decl : std::views::split(*list, ' ')) { std::string s; std::ranges::copy(decl, std::back_inserter(s)); // use range based constructor - decls_.emplace(std::move(s)); + skip_decls_.emplace(std::move(s)); } #endif // defined(__clang_major__) && __clang_major__ < 16 + decls_ = skip_decls_; list = Options.get("ExtraDeclarations"); // TODO(LLVM-17) Remove clang 15 work-around. @@ -94,7 +101,7 @@ header_exportable_declarations::header_exportable_declarations( auto b = s.begin(); auto e = std::find(b, s.end(), ' '); while (b != e) { - std::cout << "using " << std::string_view{b, e} << ";\n"; + std::cout << "using ::" << std::string_view{b, e} << ";\n"; if (e == s.end()) break; b = e + 1; @@ -104,10 +111,16 @@ header_exportable_declarations::header_exportable_declarations( #else // defined(__clang_major__) && __clang_major__ < 16 if (list) for (auto decl : std::views::split(*list, ' ')) - std::cout << "using " << std::string_view{decl.data(), decl.size()} << ";\n"; + std::cout << "using ::" << std::string_view{decl.data(), decl.size()} << ";\n"; #endif // defined(__clang_major__) && __clang_major__ < 16 } +header_exportable_declarations::~header_exportable_declarations() { + for (const auto& name : global_decls_) + if (!skip_decls_.contains("std::" + name) && decls_.contains("std::" + name)) + std::cout << "using ::" << name << ";\n"; +} + void header_exportable_declarations::registerMatchers(clang::ast_matchers::MatchFinder* finder) { // there are no public names in the Standard starting with an underscore, so // no need to check the strict rules. @@ -129,10 +142,22 @@ void header_exportable_declarations::registerMatchers(clang::ast_matchers::Match .bind("header_exportable_declarations"), this); break; + case FileType::CHeader: + // For C headers of the std.compat two matchers are used + // - The cheader matcher; in libc++ these are never split in multiple + // headers so limiting the declarations to that header works. + // - The header.h; where the declarations of this header are provided + // is not specified and depends on the libc used. Therefore it is not + // possible to restrict the location in a portable way. + finder->addMatcher(namedDecl().bind("cheader_exportable_declarations"), this); + + [[fallthrough]]; case FileType::ModulePartition: + case FileType::CompatModulePartition: finder->addMatcher(namedDecl(isExpansionInFileMatching(filename_)).bind("header_exportable_declarations"), this); break; case FileType::Module: + case FileType::CompatModule: finder->addMatcher(namedDecl().bind("header_exportable_declarations"), this); break; case header_exportable_declarations::FileType::Unknown: @@ -156,6 +181,9 @@ void header_exportable_declarations::registerMatchers(clang::ast_matchers::Match /// * cstddef has bitwise operators for the type \c byte /// * exception has equality operators for the type \c exception_ptr /// * initializer_list has the functions \c begin and \c end +/// +/// \warning In some cases the returned name can be an empty string. +/// The cause has not been investigated. static std::string get_qualified_name(const clang::NamedDecl& decl) { std::string result = decl.getQualifiedNameAsString(); @@ -166,10 +194,6 @@ static std::string get_qualified_name(const clang::NamedDecl& decl) { } static bool is_viable_declaration(const clang::NamedDecl* decl) { - // Declarations nested in records are automatically exported with the record itself. - if (!decl->getDeclContext()->isNamespace()) - return false; - // Declarations that are a subobject of a friend Declaration are automatically exported with the record itself. if (decl->getFriendObjectKind() != clang::Decl::FOK_None) return false; @@ -199,13 +223,21 @@ static bool is_viable_declaration(const clang::NamedDecl* decl) { /// Returns the name is a reserved name. /// /// Detected reserved names are names starting with __ or _[A-Z]. -/// These names can be in the namespace std or any namespace inside std. For -/// example std::ranges contains reserved names to implement the Niebloids. +/// These names can be in the global namespace, std namespace or any namespace +/// inside std. For example, std::ranges contains reserved names to implement +/// the Niebloids. /// -/// This test misses 2 candidates which are not used in libc++ +/// This test misses candidates which are not used in libc++ /// * any identifier with two underscores not at the start -/// * a name with a leading underscore in the global namespace -bool is_reserved_name(const std::string& name) { +bool is_reserved_name(std::string_view name) { + if (name.starts_with("_")) { + // This is a public name declared in cstdlib. + if (name == "_Exit") + return false; + + return name.size() > 1 && (name[1] == '_' || std::isupper(name[1])); + } + std::size_t pos = name.find("::_"); if (pos == std::string::npos) return false; @@ -213,27 +245,73 @@ bool is_reserved_name(const std::string& name) { if (pos + 3 > name.size()) return false; + // This is a public name declared in cstdlib. + if (name == "std::_Exit") + return false; + return name[pos + 3] == '_' || std::isupper(name[pos + 3]); } +/// Some declarations in the global namespace are exported from the std module. +static bool is_global_name_exported_by_std_module(std::string_view name) { + static const std::set valid{ + "operator delete", "operator delete[]", "operator new", "operator new[]"}; + return valid.contains(name); +} + +static bool is_valid_declaration_context( + const clang::NamedDecl& decl, std::string_view name, header_exportable_declarations::FileType file_type) { + if (decl.getDeclContext()->isNamespace()) + return true; + + if (is_global_name_exported_by_std_module(name)) + return true; + + return file_type != header_exportable_declarations::FileType::Header; +} + +static bool is_module(header_exportable_declarations::FileType file_type) { + switch (file_type) { + case header_exportable_declarations::FileType::Module: + case header_exportable_declarations::FileType::ModulePartition: + case header_exportable_declarations::FileType::CompatModule: + case header_exportable_declarations::FileType::CompatModulePartition: + return true; + + case header_exportable_declarations::FileType::Header: + case header_exportable_declarations::FileType::CHeader: + return false; + + case header_exportable_declarations::FileType::Unknown: + llvm::errs() << "This should be unreachable.\n"; + break; + } +} + void header_exportable_declarations::check(const clang::ast_matchers::MatchFinder::MatchResult& result) { if (const auto* decl = result.Nodes.getNodeAs("header_exportable_declarations"); decl != nullptr) { if (!is_viable_declaration(decl)) return; std::string name = get_qualified_name(*decl); + if (name.empty()) + return; + if (is_reserved_name(name)) return; // For modules only take the declarations exported. - if (file_type_ == FileType::ModulePartition || file_type_ == FileType::Module) + if (is_module(file_type_)) if (decl->getModuleOwnershipKind() != clang::Decl::ModuleOwnershipKind::VisibleWhenImported) return; + if (!is_valid_declaration_context(*decl, name, file_type_)) + return; + if (decls_.contains(name)) { // For modules avoid exporting the same named declaration twice. For // header files this is common and valid. - if (file_type_ == FileType::ModulePartition) + if (file_type_ == FileType::ModulePartition || file_type_ == FileType::CompatModulePartition) // After the warning the script continues. // The test will fail since modules have duplicated entries and headers not. llvm::errs() << "Duplicated export of '" << name << "'.\n"; @@ -241,8 +319,30 @@ void header_exportable_declarations::check(const clang::ast_matchers::MatchFinde return; } - std::cout << "using " << std::string{name} << ";\n"; + // For named declarations in std this is valid + // using std::foo; + // for named declarations it is invalid to use + // using bar; + // Since fully qualifying named declarations in the std namespace is valid + // using fully qualified names unconditionally. + std::cout << "using ::" << std::string{name} << ";\n"; decls_.insert(name); + } else if (const auto* decl = result.Nodes.getNodeAs("cheader_exportable_declarations"); + decl != nullptr) { + if (decl->getDeclContext()->isNamespace()) + return; + + if (!is_viable_declaration(decl)) + return; + + std::string name = get_qualified_name(*decl); + if (is_reserved_name(name)) + return; + + if (global_decls_.contains(name)) + return; + + global_decls_.insert(name); } } diff --git a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.hpp b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.hpp index 119bcd3f6f2f..5d9e0f3ef9c3 100644 --- a/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.hpp +++ b/libcxx/test/tools/clang_tidy_checks/header_exportable_declarations.hpp @@ -18,15 +18,47 @@ namespace libcpp { class header_exportable_declarations : public clang::tidy::ClangTidyCheck { public: explicit header_exportable_declarations(llvm::StringRef, clang::tidy::ClangTidyContext*); + ~header_exportable_declarations(); void registerMatchers(clang::ast_matchers::MatchFinder*) override; void check(const clang::ast_matchers::MatchFinder::MatchResult&) override; - enum class FileType { Header, ModulePartition, Module, Unknown }; + enum class FileType { + // std module specific + Header, + CompatModulePartition, + Module, + // std.compat module specific + CHeader, + ModulePartition, + CompatModule, + // invalid value + Unknown + }; private: llvm::StringRef filename_; FileType file_type_; llvm::StringRef extra_header_; std::set decls_; + std::set global_decls_; + + // The named declarations in .h C headers are "tricky". On POSIX + // systems these headers contain POSIX specific functions that do not + // use a reserved name. For example, fmemopen is provided by stdio.h. + // We filter the names that should be provided by the headers as follows: + // - record all named declarations the global namespace + // - wait until the header is completely processed + // - every named declaration in the global namespace that has a matching + // "export" in the std namespace is exported. + // + // The only place where we can do the above while ensuring that all + // the declarations in the header have been seen is in the clang tidy + // plugin's destructor. + // + // It is possible to skip some declarations in the std namespace, + // these are added to decls_ before processing. To differentiate + // between a skipped declaration and a real declaration the skipped + // declarations are recorded in an extra variable. + std::set skip_decls_; }; } // namespace libcpp diff --git a/libcxx/utils/CMakeLists.txt b/libcxx/utils/CMakeLists.txt index 7e597f632b6c..19bb9851c867 100644 --- a/libcxx/utils/CMakeLists.txt +++ b/libcxx/utils/CMakeLists.txt @@ -7,9 +7,19 @@ add_custom_target(libcxx-generate-std-clang-module-header COMMENT "Generate the <__std_clang_module> header") add_custom_target(libcxx-generate-std-cppm-in-file - COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/generate_std_cppm_in.py" + COMMAND + "${Python3_EXECUTABLE}" + "${LIBCXX_SOURCE_DIR}/utils/generate_libcxx_cppm_in.py" + "std" COMMENT "Generate the std.cppm.in file") +add_custom_target(libcxx-generate-std-compat-cppm-in-file + COMMAND + "${Python3_EXECUTABLE}" + "${LIBCXX_SOURCE_DIR}/utils/generate_libcxx_cppm_in.py" + "std.compat" + COMMENT "Generate the std.compat.cppm.in file") + add_custom_target(libcxx-generate-extended-grapheme-cluster-tables COMMAND "${Python3_EXECUTABLE}" @@ -48,6 +58,7 @@ add_custom_target(libcxx-generate-files DEPENDS libcxx-generate-feature-test-macros libcxx-generate-std-clang-module-header libcxx-generate-std-cppm-in-file + libcxx-generate-std-compat-cppm-in-file libcxx-generate-extended-grapheme-cluster-tables libcxx-generate-extended-grapheme-cluster-tests libcxx-generate-escaped-output-table diff --git a/libcxx/utils/generate_std_cppm_in.py b/libcxx/utils/generate_libcxx_cppm_in.py similarity index 51% rename from libcxx/utils/generate_std_cppm_in.py rename to libcxx/utils/generate_libcxx_cppm_in.py index 242134773e68..f957406778d3 100644 --- a/libcxx/utils/generate_std_cppm_in.py +++ b/libcxx/utils/generate_libcxx_cppm_in.py @@ -7,20 +7,22 @@ # ===----------------------------------------------------------------------===## import os.path +import sys from libcxx.header_information import module_headers from libcxx.header_information import header_restrictions from libcxx.header_information import headers_not_available -libcxx_module_directory = os.path.join( - os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "modules" -) -with open( - os.path.join(libcxx_module_directory, "std.cppm.in"), "w" -) as std_module_cpp_in: - std_module_cpp_in.write( - """\ +def write_file(module): + libcxx_module_directory = os.path.join( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))), "modules" + ) + with open( + os.path.join(libcxx_module_directory, f"{module}.cppm.in"), "w" + ) as module_cpp_in: + module_cpp_in.write( + """\ // -*- C++ -*- //===----------------------------------------------------------------------===// // @@ -31,7 +33,7 @@ with open( //===----------------------------------------------------------------------===// // WARNING, this entire header is generated by -// utils/generate_std_cppm_in.py +// utils/generate_libcxx_cppm_in.py // DO NOT MODIFY! module; @@ -41,33 +43,46 @@ module; // The headers of Table 24: C++ library headers [tab:headers.cpp] // and the headers of Table 25: C++ headers for C library facilities [tab:headers.cpp.c] """ - ) - for header in module_headers: - if header in header_restrictions: - std_module_cpp_in.write( - f"""\ + ) + for header in module_headers: + if header in header_restrictions: + module_cpp_in.write( + f"""\ #if {header_restrictions[header]} # include <{header}> #endif """ - ) - else: - std_module_cpp_in.write(f"#include <{header}>\n") + ) + else: + module_cpp_in.write(f"#include <{header}>\n") - std_module_cpp_in.write("\n// *** Headers not yet available ***\n") - for header in sorted(headers_not_available): - std_module_cpp_in.write( - f"""\ + module_cpp_in.write("\n// *** Headers not yet available ***\n") + for header in sorted(headers_not_available): + module_cpp_in.write( + f"""\ #if __has_include(<{header}>) -# error "update the header information for <{header}> in libcxx/utils/generate_std_cppm_in.py" -#endif // __has_include(<{header}>) +# error "please update the header information for <{header}> in headers_not_available in utils/libcxx/header_information.py" +#endif // __has_include(<{header}>) """ - ) + ) - std_module_cpp_in.write( - """ -export module std; + module_cpp_in.write( + f""" +export module {module}; @LIBCXX_MODULE_STD_INCLUDE_SOURCES@ +{'@LIBCXX_MODULE_STD_COMPAT_INCLUDE_SOURCES@' if module == 'std.compat' else ''}""" + ) + + +if __name__ == "__main__": + if len(sys.argv) != 2 or (sys.argv[1] != "std" and sys.argv[1] != "std.compat"): + sys.stderr.write( + f"""\ +Usage: +{os.path.basename(__file__)} (std|std.compat) """ - ) + ) + sys.exit(1) + + write_file(sys.argv[1]) diff --git a/libcxx/utils/libcxx/test/features.py b/libcxx/utils/libcxx/test/features.py index ccabb48833f1..5e854917e6ef 100644 --- a/libcxx/utils/libcxx/test/features.py +++ b/libcxx/utils/libcxx/test/features.py @@ -198,6 +198,18 @@ DEFAULT_FEATURES = [ """, ), ), + # Tests that require 64-bit architecture + Feature( + name="32-bit-pointer", + when=lambda cfg: sourceBuilds( + cfg, + """ + int main(int, char**) { + static_assert(sizeof(void *) == 4); + } + """, + ), + ), # TODO: Remove this feature once compiler-rt includes __atomic_is_lockfree() # on all supported platforms. Feature( @@ -567,16 +579,4 @@ DEFAULT_FEATURES += [ cfg.available_features, ), ), - # Tests that require 64-bit architecture - Feature( - name="32-bit-pointer", - when=lambda cfg: sourceBuilds( - cfg, - """ - int main(int, char**) { - static_assert(sizeof(void *) == 4); - } - """, - ), - ), ] diff --git a/libcxx/utils/libcxx/test/modules.py b/libcxx/utils/libcxx/test/modules.py new file mode 100644 index 000000000000..deaac450381c --- /dev/null +++ b/libcxx/utils/libcxx/test/modules.py @@ -0,0 +1,302 @@ +# ===----------------------------------------------------------------------===## +# +# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# ===----------------------------------------------------------------------===## + +from libcxx.header_information import module_headers +from libcxx.header_information import header_restrictions +from dataclasses import dataclass + +### SkipDeclarations + +# Ignore several declarations found in the includes. +# +# Part of these items are bugs other are not yet implemented features. +SkipDeclarations = dict() + +# See comment in the header. +SkipDeclarations["cuchar"] = ["std::mbstate_t", "std::size_t"] + +# Not in the synopsis. +SkipDeclarations["cwchar"] = ["std::FILE"] + +# The operators are added for private types like __iom_t10. +SkipDeclarations["iomanip"] = ["std::operator<<", "std::operator>>"] + +SkipDeclarations["iosfwd"] = ["std::ios_base", "std::vector"] + +# This header also provides declarations in the namespace that might be +# an error. +SkipDeclarations["filesystem"] = [ + "std::filesystem::operator==", + "std::filesystem::operator!=", +] + +# This is a specialization for a private type +SkipDeclarations["iterator"] = ["std::pointer_traits"] + +# TODO MODULES +# This definition is declared in string and defined in istream +# This declaration should be part of string +SkipDeclarations["istream"] = ["std::getline"] + +# P1614 (at many places) and LWG3519 too. +SkipDeclarations["random"] = [ + "std::operator!=", + # LWG3519 makes these hidden friends. + # Note the older versions had the requirement of these operations but not in + # the synopsis. + "std::operator<<", + "std::operator>>", + "std::operator==", +] + +# Declared in the forward header since std::string uses std::allocator +SkipDeclarations["string"] = ["std::allocator"] +# TODO MODULES remove zombie names +# https://libcxx.llvm.org/Status/Cxx20.html#note-p0619 +SkipDeclarations["memory"] = [ + "std::return_temporary_buffer", + "std::get_temporary_buffer", +] + +# TODO MODULES this should be part of ios instead +SkipDeclarations["streambuf"] = ["std::basic_ios"] + +# include/__type_traits/is_swappable.h +SkipDeclarations["type_traits"] = [ + "std::swap", + # TODO MODULES gotten through __functional/unwrap_ref.h + "std::reference_wrapper", +] + +### ExtraDeclarations + +# Add declarations in headers. +# +# Some headers have their defines in a different header, which may have +# additional declarations. +ExtraDeclarations = dict() +# This declaration is in the ostream header. +ExtraDeclarations["system_error"] = ["std::operator<<"] + +### ExtraHeader + +# Adds extra headers file to scan +# +# Some C++ headers in libc++ are stored in multiple physical files. There is a +# pattern to find these files. However there are some exceptions these are +# listed here. +ExtraHeader = dict() +# locale has a file and not a subdirectory +ExtraHeader["locale"] = "v1/__locale$" +ExtraHeader["thread"] = "v1/__threading_support$" +ExtraHeader["ranges"] = "v1/__fwd/subrange.h$" + +# The extra header is needed since two headers are required to provide the +# same definition. +ExtraHeader["functional"] = "v1/__compare/compare_three_way.h$" + + +# newline needs to be escaped for the module partition output. +nl = "\\\\n" + + +@dataclass +class module_test_generator: + tmp_prefix: str + module_path: str + clang_tidy: str + clang_tidy_plugin: str + compiler: str + compiler_flags: str + + def write_lit_configuration(self): + print( + f"""\ +// UNSUPPORTED: c++03, c++11, c++14, c++17 +// UNSUPPORTED: libcpp-has-no-std-modules +// UNSUPPORTED: clang-modules-build + +// REQUIRES: has-clang-tidy + +// The GCC compiler flags are not always compatible with clang-tidy. +// UNSUPPORTED: gcc + +// RUN: echo -n > {self.tmp_prefix}.all_partitions +""" + ) + + def process_module_partition(self, header, is_c_header): + # Some headers cannot be included when a libc++ feature is disabled. + # In that case include the header conditionally. The header __config + # ensures the libc++ feature macros are available. + if header in header_restrictions: + include = ( + f"#include <__config>{nl}" + f"#if {header_restrictions[header]}{nl}" + f"# include <{header}>{nl}" + f"#endif{nl}" + ) + elif header == "chrono": + # When localization is disabled the header string is not included. + # When string is included chrono's operator""s is a named declaration + # using std::chrono_literals::operator""s; + # else it is a named declaration + # using std::operator""s; + # TODO MODULES investigate why + include = f"#include {nl}#include {nl}" + else: + include = f"#include <{header}>{nl}" + + module_files = f'#include \\"{self.module_path}/std/{header}.inc\\"{nl}' + if is_c_header: + module_files += ( + f'#include \\"{self.module_path}/std.compat/{header}.inc\\"{nl}' + ) + + # Generate a module partition for the header module includes. This + # makes it possible to verify that all headers export all their + # named declarations. + print( + '// RUN: echo -e "' + f"module;{nl}" + f"{include}{nl}" + f"{nl}" + f"// Use __libcpp_module_
to ensure that modules{nl}" + f"// are not named as keywords or reserved names.{nl}" + f"export module std:__libcpp_module_{header};{nl}" + f"{module_files}" + f'" > {self.tmp_prefix}.{header}.cppm' + ) + + # Extract the information of the module partition using lang-tidy + print( + f"// RUN: {self.clang_tidy} {self.tmp_prefix}.{header}.cppm " + " --checks='-*,libcpp-header-exportable-declarations' " + " -config='{CheckOptions: [ " + " {" + " key: libcpp-header-exportable-declarations.Filename, " + f" value: {header}.inc" + " }, {" + " key: libcpp-header-exportable-declarations.FileType, " + f" value: {'CompatModulePartition' if is_c_header else 'ModulePartition'}" + " }, " + " ]}' " + f"--load={self.clang_tidy_plugin} " + f"-- {self.compiler_flags} " + f"| sort > {self.tmp_prefix}.{header}.module" + ) + print( + f"// RUN: cat {self.tmp_prefix}.{header}.module >> {self.tmp_prefix}.all_partitions" + ) + + return include + + def process_header(self, header, include, is_c_header): + # Dump the information as found in the module by using the header file(s). + skip_declarations = " ".join(SkipDeclarations.get(header, [])) + if skip_declarations: + skip_declarations = ( + "{" + " key: libcpp-header-exportable-declarations.SkipDeclarations, " + f' value: "{skip_declarations}" ' + "}, " + ) + + extra_declarations = " ".join(ExtraDeclarations.get(header, [])) + if extra_declarations: + extra_declarations = ( + "{" + " key: libcpp-header-exportable-declarations.ExtraDeclarations, " + f' value: "{extra_declarations}" ' + "}, " + ) + + extra_header = ExtraHeader.get(header, "") + if extra_header: + extra_header = ( + "{" + " key: libcpp-header-exportable-declarations.ExtraHeader, " + f' value: "{extra_header}" ' + "}, " + ) + + # Clang-tidy needs a file input + print(f'// RUN: echo -e "' f"{include}" f'" > {self.tmp_prefix}.{header}.cpp') + print( + f"// RUN: {self.clang_tidy} {self.tmp_prefix}.{header}.cpp " + " --checks='-*,libcpp-header-exportable-declarations' " + " -config='{CheckOptions: [ " + " {" + " key: libcpp-header-exportable-declarations.Filename, " + f" value: {header}" + " }, {" + " key: libcpp-header-exportable-declarations.FileType, " + f" value: {'CHeader' if is_c_header else 'Header'}" + " }, " + f" {skip_declarations} {extra_declarations} {extra_header}, " + " ]}' " + f"--load={self.clang_tidy_plugin} " + f"-- {self.compiler_flags} " + f"| sort > {self.tmp_prefix}.{header}.include" + ) + print( + f"// RUN: diff -u {self.tmp_prefix}.{header}.module {self.tmp_prefix}.{header}.include" + ) + + def process_module(self, module): + # Merge the data of the parts + print( + f"// RUN: sort -u -o {self.tmp_prefix}.all_partitions {self.tmp_prefix}.all_partitions" + ) + + # Dump the information as found in top-level module. + print( + f"// RUN: {self.clang_tidy} {self.module_path}/{module}.cppm " + " --checks='-*,libcpp-header-exportable-declarations' " + " -config='{CheckOptions: [ " + " {" + " key: libcpp-header-exportable-declarations.Header, " + f" value: {module}.cppm" + " }, {" + " key: libcpp-header-exportable-declarations.FileType, " + " value: Module" + " }, " + " ]}' " + f"--load={self.clang_tidy_plugin} " + f"-- {self.compiler_flags} " + f"| sort > {self.tmp_prefix}.module" + ) + + # Compare the sum of the parts with the top-level module. + print( + f"// RUN: diff -u {self.tmp_prefix}.all_partitions {self.tmp_prefix}.module" + ) + + # Basic smoke test. Import a module and try to compile when using all + # exported names. This validates the clang-tidy script does not + # accidentally add named declarations to the list that are not available. + def test_module(self, module): + print( + f"""\ +// RUN: echo 'import {module};' > {self.tmp_prefix}.compile.pass.cpp +// RUN: cat {self.tmp_prefix}.all_partitions >> {self.tmp_prefix}.compile.pass.cpp +// RUN: {self.compiler} {self.compiler_flags} -fsyntax-only {self.tmp_prefix}.compile.pass.cpp +""" + ) + + def write_test(self, module, c_headers=[]): + self.write_lit_configuration() + + # Validate all module parts. + for header in module_headers: + is_c_header = header in c_headers + include = self.process_module_partition(header, is_c_header) + self.process_header(header, include, is_c_header) + + self.process_module(module) + self.test_module(module) diff --git a/libcxxabi/src/cxa_personality.cpp b/libcxxabi/src/cxa_personality.cpp index 4570d0b5beb2..4b6c4edbc266 100644 --- a/libcxxabi/src/cxa_personality.cpp +++ b/libcxxabi/src/cxa_personality.cpp @@ -660,10 +660,9 @@ static void scan_eh_tab(scan_results &results, _Unwind_Action actions, // dwarf emission // Parse LSDA header. uint8_t lpStartEncoding = *lsda++; - const uint8_t* lpStart = - (const uint8_t*)readEncodedPointer(&lsda, lpStartEncoding, base); - if (lpStart == 0) - lpStart = (const uint8_t*)funcStart; + const uint8_t* lpStart = lpStartEncoding == DW_EH_PE_omit + ? (const uint8_t*)funcStart + : (const uint8_t*)readEncodedPointer(&lsda, lpStartEncoding, base); uint8_t ttypeEncoding = *lsda++; if (ttypeEncoding != DW_EH_PE_omit) { diff --git a/libcxxabi/test/native/x86_64/lpstart-zero.pass.sh.s b/libcxxabi/test/native/x86_64/lpstart-zero.pass.sh.s new file mode 100644 index 000000000000..ea9dd0104c08 --- /dev/null +++ b/libcxxabi/test/native/x86_64/lpstart-zero.pass.sh.s @@ -0,0 +1,109 @@ +# RUN: %{cxx} %{flags} %s %{link_flags} -no-pie -o %t.exe +# RUN: %t.exe + +# REQUIRES: linux && target={{x86_64-.+}} +# UNSUPPORTED: no-exceptions + +## Check that libc++abi works correctly when LPStart address is explicitly set +## to zero. + +## This file is generated from the following C++ source code. +## +## ``` +## int main() { +## try { +## throw 42; +## } catch (...) { +## return 0; +## } +## return 1; +## } +## ``` +## The exception table is modified to use udata4 encoding for LPStart and +## sdata4 encoding for call sites. + + .text + .globl main # -- Begin function main + .p2align 4, 0x90 + .type main,@function +main: # @main +.Lfunc_begin0: + .cfi_startproc + .globl __gxx_personality_v0 + .cfi_personality 3, __gxx_personality_v0 + .cfi_lsda 27, .Lexception0 +# %bb.0: # %entry + pushq %rbp + .cfi_def_cfa_offset 16 + .cfi_offset %rbp, -16 + movq %rsp, %rbp + .cfi_def_cfa_register %rbp + subq $32, %rsp + movl $0, -4(%rbp) + movl $4, %edi + callq __cxa_allocate_exception@PLT + movq %rax, %rdi + movl $42, (%rdi) +.Ltmp0: + movq _ZTIi@GOTPCREL(%rip), %rsi + xorl %eax, %eax + movl %eax, %edx + callq __cxa_throw@PLT +.Ltmp1: + jmp .LBB0_4 +.LBB0_1: # %lpad +.Ltmp2: + movq %rax, %rcx + movl %edx, %eax + movq %rcx, -16(%rbp) + movl %eax, -20(%rbp) +# %bb.2: # %catch + movq -16(%rbp), %rdi + callq __cxa_begin_catch@PLT + movl $0, -4(%rbp) + callq __cxa_end_catch@PLT +# %bb.3: # %return + movl -4(%rbp), %eax + addq $32, %rsp + popq %rbp + .cfi_def_cfa %rsp, 8 + retq +.LBB0_4: # %unreachable +.Lfunc_end0: + .size main, .Lfunc_end0-main + .cfi_endproc + + .section .gcc_except_table,"a",@progbits + .p2align 2, 0x0 +GCC_except_table0: +.Lexception0: + .byte 3 # @LPStart Encoding = udata4 + .long 0 + .byte 155 # @TType Encoding = indirect pcrel sdata4 + .uleb128 .Lttbase0-.Lttbaseref0 +.Lttbaseref0: + .byte 11 # Call site Encoding = udata4 + .uleb128 .Lcst_end0-.Lcst_begin0 +.Lcst_begin0: + .long .Lfunc_begin0-.Lfunc_begin0 # >> Call Site 1 << + .long .Ltmp0-.Lfunc_begin0 # Call between .Lfunc_begin0 and .Ltmp0 + .long 0 # has no landing pad + .byte 0 # On action: cleanup + .long .Ltmp0-.Lfunc_begin0 # >> Call Site 2 << + .long .Ltmp1-.Ltmp0 # Call between .Ltmp0 and .Ltmp1 + .long .Ltmp2 + .byte 1 # On action: 1 + .long .Ltmp1-.Lfunc_begin0 # >> Call Site 3 << + .long .Lfunc_end0-.Ltmp1 # Call between .Ltmp1 and .Lfunc_end0 + .long 0 # has no landing pad + .byte 0 # On action: cleanup +.Lcst_end0: + .byte 1 # >> Action Record 1 << + # Catch TypeInfo 1 + .byte 0 # No further actions + .p2align 2, 0x0 + # >> Catch TypeInfos << + .long 0 # TypeInfo 1 +.Lttbase0: + .p2align 2, 0x0 + # -- End function diff --git a/lld/COFF/PDB.cpp b/lld/COFF/PDB.cpp index f77ff0d4eab8..8b81a71e8fb6 100644 --- a/lld/COFF/PDB.cpp +++ b/lld/COFF/PDB.cpp @@ -268,7 +268,7 @@ void PDBLinker::pdbMakeAbsolute(SmallVectorImpl &fileName) { // decide that it's a unix path if we're fairly certain. Specifically, if // it starts with a forward slash. SmallString<128> absoluteFileName = ctx.config.pdbSourcePath; - sys::path::Style guessedStyle = absoluteFileName.startswith("/") + sys::path::Style guessedStyle = absoluteFileName.starts_with("/") ? sys::path::Style::posix : sys::path::Style::windows; sys::path::append(absoluteFileName, guessedStyle, fileName); diff --git a/lld/ELF/DWARF.h b/lld/ELF/DWARF.h index e1688fef9c9e..1b9a3e3f7794 100644 --- a/lld/ELF/DWARF.h +++ b/lld/ELF/DWARF.h @@ -13,6 +13,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/DebugInfo/DWARF/DWARFContext.h" #include "llvm/Object/ELF.h" +#include namespace lld::elf { diff --git a/lld/ELF/InputFiles.cpp b/lld/ELF/InputFiles.cpp index 06a3d565deb7..cc2c5916e05c 100644 --- a/lld/ELF/InputFiles.cpp +++ b/lld/ELF/InputFiles.cpp @@ -30,6 +30,7 @@ #include "llvm/Support/RISCVAttributeParser.h" #include "llvm/Support/TarWriter.h" #include "llvm/Support/raw_ostream.h" +#include using namespace llvm; using namespace llvm::ELF; diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp index 6e972cf084cd..81468a20dfb5 100644 --- a/lld/ELF/InputSection.cpp +++ b/lld/ELF/InputSection.cpp @@ -22,6 +22,7 @@ #include "llvm/Support/xxhash.h" #include #include +#include #include using namespace llvm; @@ -909,16 +910,8 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef rels) { for (size_t i = 0, relsSize = rels.size(); i != relsSize; ++i) { const RelTy &rel = rels[i]; - RelType type = rel.getType(config->isMips64EL); - - // GCC 8.0 or earlier have a bug that they emit R_386_GOTPC relocations - // against _GLOBAL_OFFSET_TABLE_ for .debug_info. The bug has been fixed - // in 2017 (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82630), but we - // need to keep this bug-compatible code for a while. - if (emachine == EM_386 && type == R_386_GOTPC) - continue; - - uint64_t offset = rel.r_offset; + const RelType type = rel.getType(config->isMips64EL); + const uint64_t offset = rel.r_offset; uint8_t *bufLoc = buf + offset; int64_t addend = getAddend(rel); if (!RelTy::IsRela) @@ -998,25 +991,25 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef rels) { if (config->relocatable && (RelTy::IsRela || sym.type != STT_SECTION)) continue; - if (expr == R_SIZE) { - target.relocateNoSym(bufLoc, type, - SignExtend64(sym.getSize() + addend)); - continue; - } - // R_ABS/R_DTPREL and some other relocations can be used from non-SHF_ALLOC // sections. - if (expr == R_ABS || expr == R_DTPREL || expr == R_GOTPLTREL || + if (LLVM_LIKELY(expr == R_ABS) || expr == R_DTPREL || expr == R_GOTPLTREL || expr == R_RISCV_ADD) { target.relocateNoSym(bufLoc, type, SignExtend64(sym.getVA(addend))); continue; } + if (expr == R_SIZE) { + target.relocateNoSym(bufLoc, type, + SignExtend64(sym.getSize() + addend)); + continue; + } + std::string msg = getLocation(offset) + ": has non-ABS relocation " + toString(type) + " against symbol '" + toString(sym) + "'"; - if (expr != R_PC) { - error(msg); + if (expr != R_PC && !(emachine == EM_386 && type == R_386_GOTPC)) { + errorOrWarn(msg); return; } @@ -1028,11 +1021,11 @@ void InputSection::relocateNonAlloc(uint8_t *buf, ArrayRef rels) { // address 0. For bug-compatibility, we accept them with warnings. We // know Steel Bank Common Lisp as of 2018 have this bug. // - // RELA -r stopped earlier and does not get the warning. Suppress the - // warning for REL -r as well - // (https://github.com/ClangBuiltLinux/linux/issues/1937). - if (RelTy::IsRela || !config->relocatable) - warn(msg); + // GCC 8.0 or earlier have a bug that they emit R_386_GOTPC relocations + // against _GLOBAL_OFFSET_TABLE_ for .debug_info. The bug has been fixed in + // 2017 (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=82630), but we need to + // keep this bug-compatible code for a while. + warn(msg); target.relocateNoSym( bufLoc, type, SignExtend64(sym.getVA(addend - offset - outSecOff))); diff --git a/lld/MachO/InputFiles.cpp b/lld/MachO/InputFiles.cpp index 31ed24149e78..7d0cdce9de7d 100644 --- a/lld/MachO/InputFiles.cpp +++ b/lld/MachO/InputFiles.cpp @@ -1535,7 +1535,7 @@ std::string ObjFile::sourceFile() const { StringRef sep = sys::path::get_separator(); // We don't use `path::append` here because we want an empty `dir` to result // in an absolute path. `append` would give us a relative path for that case. - if (!dir.endswith(sep)) + if (!dir.ends_with(sep)) dir += sep; return (dir + unitName).str(); } diff --git a/lld/test/ELF/debug-dead-reloc-32.s b/lld/test/ELF/debug-dead-reloc-32.s index b2708a744f28..99335b44f51c 100644 --- a/lld/test/ELF/debug-dead-reloc-32.s +++ b/lld/test/ELF/debug-dead-reloc-32.s @@ -14,6 +14,17 @@ # CHECK-NEXT: Contents of section .debug_addr: # CHECK-NEXT: 0000 00000000 +## -z dead-reloc-in-nonalloc= can override the tombstone value. +# RUN: ld.lld -z dead-reloc-in-nonalloc=.debug_loc=42 -z dead-reloc-in-nonalloc=.debug_addr=0xfffffffffffffffe %t.o -o %t1 +# RUN: llvm-objdump -s %t1 | FileCheck %s --check-prefix=OVERRIDE + +# OVERRIDE: Contents of section .debug_loc: +# OVERRIDE-NEXT: 0000 2a000000 *... +# OVERRIDE-NEXT: Contents of section .debug_ranges: +# OVERRIDE-NEXT: 0000 01000000 .... +# OVERRIDE-NEXT: Contents of section .debug_addr: +# OVERRIDE-NEXT: 0000 feffffff .... + .section .text.1,"axe" .byte 0 diff --git a/lld/test/ELF/debug-dead-reloc.s b/lld/test/ELF/debug-dead-reloc.s index fcf53205079e..cfa41e00eab0 100644 --- a/lld/test/ELF/debug-dead-reloc.s +++ b/lld/test/ELF/debug-dead-reloc.s @@ -1,4 +1,4 @@ -# REQUIRES: x86 +# REQUIRES: aarch64, x86 ## Test we resolve symbolic relocations in .debug_* sections to a tombstone ## value if the referenced symbol is discarded (--gc-sections, non-prevailing ## section group, SHF_EXCLUDE, /DISCARD/, etc). @@ -9,13 +9,18 @@ # RUN: llvm-objdump -s %t | FileCheck %s # RUN: llvm-readobj -r %t | FileCheck %s --check-prefix=REL +# RUN: echo '.globl _start; _start: bl group' | llvm-mc -filetype=obj -triple=aarch64 - -o %t.a64.o +# RUN: llvm-mc -filetype=obj -triple=aarch64 %s -o %t1.a64.o +# RUN: ld.lld --emit-relocs --gc-sections %t.a64.o %t1.a64.o %t1.a64.o -o %t.a64 +# RUN: llvm-objdump -s %t.a64 | FileCheck %s + # CHECK: Contents of section .debug_loc: # CHECK-NEXT: 0000 01000000 00000000 01000000 00000000 # CHECK: Contents of section .debug_ranges: # CHECK-NEXT: 0000 01000000 00000000 01000000 00000000 # CHECK: Contents of section .debug_addr: -# CHECK-NEXT: 0000 {{.*}}000 00000000 {{.*}}000 00000000 -# CHECK-NEXT: 0010 00000000 00000000 {{.*}}000 00000000 +# CHECK-NEXT: 0000 {{.*}}00 00000000 {{.*}}00 00000000 +# CHECK-NEXT: 0010 00000000 00000000 {{.*}}00 00000000 # CHECK: Contents of section .debug_foo: # CHECK-NEXT: 0000 00000000 00000000 08000000 00000000 # CHECK-NEXT: 0010 00000000 00000000 08000000 00000000 diff --git a/lld/test/ELF/emulation-amdgpu.s b/lld/test/ELF/emulation-amdgpu.s index 707f0aeb909e..2d9ae52548b4 100644 --- a/lld/test/ELF/emulation-amdgpu.s +++ b/lld/test/ELF/emulation-amdgpu.s @@ -1,10 +1,16 @@ # REQUIRES: amdgpu -# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa --amdhsa-code-object-version=4 %s -o %t.o # RUN: ld.lld %t.o -o %t -# RUN: llvm-readobj --file-headers %t | FileCheck %s +# RUN: llvm-readobj --file-headers %t | FileCheck --check-prefixes=CHECK,HSA4 %s # RUN: ld.lld -m elf64_amdgpu %t.o -o %t -# RUN: llvm-readobj --file-headers %t | FileCheck %s +# RUN: llvm-readobj --file-headers %t | FileCheck --check-prefixes=CHECK,HSA4 %s + +# RUN: llvm-mc -filetype=obj -triple=amdgcn-amd-amdhsa --amdhsa-code-object-version=5 %s -o %t.o +# RUN: ld.lld %t.o -o %t +# RUN: llvm-readobj --file-headers %t | FileCheck --check-prefixes=CHECK,HSA5 %s +# RUN: ld.lld -m elf64_amdgpu %t.o -o %t +# RUN: llvm-readobj --file-headers %t | FileCheck --check-prefixes=CHECK,HSA5 %s # CHECK: ElfHeader { # CHECK-NEXT: Ident { @@ -13,8 +19,9 @@ # CHECK-NEXT: DataEncoding: LittleEndian (0x1) # CHECK-NEXT: FileVersion: 1 # CHECK-NEXT: OS/ABI: AMDGPU_HSA (0x40) -# CHECK-NEXT: ABIVersion: 2 -# CHECK-NEXT: Unused: (00 00 00 00 00 00 00) +# HSA4: ABIVersion: 2 +# HSA5: ABIVersion: 3 +# CHECK: Unused: (00 00 00 00 00 00 00) # CHECK-NEXT: } # CHECK-NEXT: Type: Executable (0x2) # CHECK-NEXT: Machine: EM_AMDGPU (0xE0) diff --git a/lld/test/ELF/i386-debug-noabs.test b/lld/test/ELF/i386-debug-noabs.test index 486cc8d9fbed..02ef238dd148 100644 --- a/lld/test/ELF/i386-debug-noabs.test +++ b/lld/test/ELF/i386-debug-noabs.test @@ -1,7 +1,8 @@ # REQUIRES: x86 # RUN: yaml2obj %s -o %t.o -# RUN: ld.lld %t.o -o /dev/null --entry 0 --fatal-warnings +# RUN: ld.lld %t.o -o /dev/null --entry 0 2>&1 | FileCheck %s +# CHECK: warning: {{.*}}:(.debug_info+0x41f): has non-ABS relocation R_386_GOTPC against symbol '_GLOBAL_OFFSET_TABLE_' ## This is for https://bugs.llvm.org//show_bug.cgi?id=34852. GCC 8.0 or ## earlier have a bug which creates non-absolute R_386_GOTPC relocations diff --git a/lld/test/ELF/non-abs-reloc.s b/lld/test/ELF/non-abs-reloc.s index 04f583ab2534..42b5f8fec1c4 100644 --- a/lld/test/ELF/non-abs-reloc.s +++ b/lld/test/ELF/non-abs-reloc.s @@ -15,7 +15,8 @@ // DISASM-NEXT: 6: call{{.}} 0x5 /// There is currently no error for -r. See also https://github.com/ClangBuiltLinux/linux/issues/1937 -// RUN: ld.lld -T lds -r a.o -o /dev/null --fatal-warnings +// RUN: ld.lld -T lds -r a.o -o /dev/null 2>&1 | FileCheck %s --check-prefix=REL-R --implicit-check-not=warning: +// REL-R: warning: {{.*}}:(.nonalloc1+0xa): has non-ABS relocation R_386_PC32 against symbol '' // RUN: llvm-mc -filetype=obj -triple=x86_64 asm -o b.o // RUN: ld.lld -T lds b.o -o b 2>&1 | FileCheck %s --check-prefix=CHECK2 --implicit-check-not=warning: diff --git a/lld/test/ELF/x86-64-dyn-rel-error.s b/lld/test/ELF/x86-64-dyn-rel-error.s index 8f41f1493dde..edc2875c6fa6 100644 --- a/lld/test/ELF/x86-64-dyn-rel-error.s +++ b/lld/test/ELF/x86-64-dyn-rel-error.s @@ -1,17 +1,43 @@ -// REQUIRES: x86 -// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o -// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/shared.s -o %t2.o -// RUN: ld.lld %t2.o -shared -o %t2.so -// RUN: not ld.lld -shared %t.o %t2.so -o /dev/null 2>&1 | FileCheck %s +# REQUIRES: x86 +# RUN: llvm-mc -filetype=obj -triple=x86_64 %s -o %t.o +# RUN: llvm-mc -filetype=obj -triple=x86_64 %p/Inputs/shared.s -o %t2.o +# RUN: ld.lld %t2.o -shared -o %t2.so --threads=1 +# RUN: not ld.lld -pie %t.o %t2.so -o /dev/null --threads=1 2>&1 | FileCheck %s +# RUN: not ld.lld -shared %t.o %t2.so -o /dev/null --threads=1 2>&1 | FileCheck %s --check-prefixes=CHECK,SHARED - .global _start -_start: - .data - .long zed +# CHECK: error: relocation R_X86_64_32 cannot be used against symbol 'zed'; recompile with -fPIC +# CHECK-NEXT: >>> defined in {{.*}}.so +# CHECK-NEXT: >>> referenced by {{.*}}.o:(.data+0x0) +# CHECK-EMPTY: +# CHECK-NEXT: error: relocation R_X86_64_PC32 cannot be used against symbol 'zed'; recompile with -fPIC +# CHECK-NEXT: >>> defined in {{.*}}.so +# CHECK-NEXT: >>> referenced by {{.*}}.o:(.data+0x4) +# CHECK-EMPTY: +# CHECK-NEXT: error: relocation R_X86_64_64 cannot be used against symbol '_start'; recompile with -fPIC +# SHARED: error: relocation R_X86_64_64 cannot be used against symbol 'main'; recompile with -fPIC +# SHARED: error: relocation R_X86_64_64 cannot be used against symbol 'data'; recompile with -fPIC +# CHECK-NOT: error: + +# RUN: ld.lld --noinhibit-exec %t.o %t2.so -o /dev/null 2>&1 | FileCheck --check-prefix=WARN %s +# RUN: not ld.lld --export-dynamic --unresolved-symbols=ignore-all %t.o -o /dev/null 2>&1 | FileCheck --check-prefix=WARN %s -// CHECK: error: relocation R_X86_64_32 cannot be used against symbol 'zed'; recompile with -fPIC +# WARN: relocation R_X86_64_32 cannot be used against symbol 'zed'; recompile with -fPIC +# WARN: relocation R_X86_64_PC32 cannot be used against symbol 'zed'; recompile with -fPIC + + .global _start, main, data + .type main, @function + .type data, @object +_start: + ret +main: + ret -// RUN: ld.lld --noinhibit-exec %t.o %t2.so -o /dev/null 2>&1 | FileCheck --check-prefix=WARN %s -// RUN: not ld.lld --export-dynamic --unresolved-symbols=ignore-all %t.o -o /dev/null 2>&1 | FileCheck --check-prefix=WARN %s +.data +data: +.long zed +.long zed - . -// WARN: relocation R_X86_64_32 cannot be used against symbol 'zed'; recompile with -fPIC +.rodata +.quad _start +.quad main +.quad data diff --git a/lld/test/ELF/x86-64-dyn-rel-error2.s b/lld/test/ELF/x86-64-dyn-rel-error2.s deleted file mode 100644 index 853e61faef68..000000000000 --- a/lld/test/ELF/x86-64-dyn-rel-error2.s +++ /dev/null @@ -1,14 +0,0 @@ -// REQUIRES: x86 -// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %s -o %t.o -// RUN: llvm-mc -filetype=obj -triple=x86_64-pc-linux %p/Inputs/shared.s -o %t2.o -// RUN: ld.lld %t2.o -shared -o %t2.so -// RUN: not ld.lld -shared %t.o %t2.so -o /dev/null 2>&1 | FileCheck %s - -// CHECK: error: relocation R_X86_64_PC32 cannot be used against symbol 'zed'; recompile with -fPIC -// CHECK: >>> defined in {{.*}}.so -// CHECK: >>> referenced by {{.*}}.o:(.data+0x0) - - .global _start -_start: - .data - .long zed - . diff --git a/lld/test/ELF/x86-64-reloc-32.s b/lld/test/ELF/x86-64-reloc-32.s index 70a46301ad24..bf2fc26a5820 100644 --- a/lld/test/ELF/x86-64-reloc-32.s +++ b/lld/test/ELF/x86-64-reloc-32.s @@ -1,9 +1,6 @@ # REQUIRES: x86 # RUN: rm -rf %t && split-file %s %t - -## Check recompile with -fPIC error message # RUN: llvm-mc -filetype=obj -triple=x86_64-unknown-linux %t/shared.s -o %t/shared.o -# RUN: not ld.lld -shared %t/shared.o -o /dev/null 2>&1 | FileCheck %s # CHECK: error: relocation R_X86_64_32 cannot be used against symbol '_shared'; recompile with -fPIC # CHECK: >>> defined in {{.*}} diff --git a/lldb/include/lldb/Core/Address.h b/lldb/include/lldb/Core/Address.h index b19e69442754..725b5d9f91d3 100644 --- a/lldb/include/lldb/Core/Address.h +++ b/lldb/include/lldb/Core/Address.h @@ -14,6 +14,8 @@ #include "lldb/lldb-private-enumerations.h" #include "lldb/lldb-types.h" +#include "llvm/ADT/StringRef.h" + #include #include @@ -237,6 +239,12 @@ public: /// contains the address, otherwise dumping the range that contains the /// address. /// + /// \param[in] pattern + /// An optional regex pattern to match against the description. If + /// specified, parts of the description matching this pattern may be + /// highlighted or processed differently. If this parameter is an empty + /// string or not provided, no highlighting is applied. + /// /// \return /// Returns \b true if the address was able to be displayed. /// File and load addresses may be unresolved and it may not be @@ -246,8 +254,8 @@ public: /// \see Address::DumpStyle bool Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, DumpStyle fallback_style = DumpStyleInvalid, - uint32_t addr_byte_size = UINT32_MAX, - bool all_ranges = false) const; + uint32_t addr_byte_size = UINT32_MAX, bool all_ranges = false, + llvm::StringRef pattern = "") const; AddressClass GetAddressClass() const; diff --git a/lldb/include/lldb/Core/Debugger.h b/lldb/include/lldb/Core/Debugger.h index e4ee94809cf1..c6d603ca5dcd 100644 --- a/lldb/include/lldb/Core/Debugger.h +++ b/lldb/include/lldb/Core/Debugger.h @@ -321,6 +321,10 @@ public: llvm::StringRef GetAutosuggestionAnsiSuffix() const; + llvm::StringRef GetRegexMatchAnsiPrefix() const; + + llvm::StringRef GetRegexMatchAnsiSuffix() const; + bool GetShowDontUsePoHint() const; bool GetUseSourceCache() const; diff --git a/lldb/include/lldb/Symbol/Symbol.h b/lldb/include/lldb/Symbol/Symbol.h index 44a2d560010f..e6c0b495bcf2 100644 --- a/lldb/include/lldb/Symbol/Symbol.h +++ b/lldb/include/lldb/Symbol/Symbol.h @@ -174,8 +174,8 @@ public: void SetFlags(uint32_t flags) { m_flags = flags; } - void GetDescription(Stream *s, lldb::DescriptionLevel level, - Target *target) const; + void GetDescription(Stream *s, lldb::DescriptionLevel level, Target *target, + llvm::StringRef pattern = "") const; bool IsSynthetic() const { return m_is_synthetic; } diff --git a/lldb/include/lldb/Symbol/SymbolContext.h b/lldb/include/lldb/Symbol/SymbolContext.h index b0f5ffead2a1..26f3bac09a96 100644 --- a/lldb/include/lldb/Symbol/SymbolContext.h +++ b/lldb/include/lldb/Symbol/SymbolContext.h @@ -145,13 +145,19 @@ public: /// is dumped if this flag is \b true, otherwise the line info /// of the actual inlined function is dumped. /// + /// \param[in] pattern + /// An optional regex pattern to match against the stop context + /// description. If specified, parts of the description matching this + /// pattern may be highlighted or processed differently. If this parameter + /// is an empty string or not provided, no highlighting is applied. + /// /// \return /// \b true if some text was dumped, \b false otherwise. bool DumpStopContext(Stream *s, ExecutionContextScope *exe_scope, const Address &so_addr, bool show_fullpaths, bool show_module, bool show_inlined_frames, - bool show_function_arguments, - bool show_function_name) const; + bool show_function_arguments, bool show_function_name, + llvm::StringRef pattern = "") const; /// Get the address range contained within a symbol context. /// @@ -217,8 +223,8 @@ public: /// The symbol that was found, or \b nullptr if none was found. const Symbol *FindBestGlobalDataSymbol(ConstString name, Status &error); - void GetDescription(Stream *s, lldb::DescriptionLevel level, - Target *target) const; + void GetDescription(Stream *s, lldb::DescriptionLevel level, Target *target, + llvm::StringRef pattern = "") const; uint32_t GetResolvedMask() const; diff --git a/lldb/include/lldb/Target/Process.h b/lldb/include/lldb/Target/Process.h index 4646e3070cf1..24c599e044c7 100644 --- a/lldb/include/lldb/Target/Process.h +++ b/lldb/include/lldb/Target/Process.h @@ -558,7 +558,10 @@ public: /// /// Subclasses that override this method should always call this superclass /// method. - virtual void Finalize(); + /// If you are running Finalize in your Process subclass Destructor, pass + /// \b true. If we are in the destructor, shared_from_this will no longer + /// work, so we have to avoid doing anything that might trigger that. + virtual void Finalize(bool destructing); /// Return whether this object is valid (i.e. has not been finalized.) /// @@ -3079,6 +3082,11 @@ protected: /// This is set at the beginning of Process::Finalize() to stop functions /// from looking up or creating things during or after a finalize call. std::atomic m_finalizing; + // When we are "Finalizing" we need to do some cleanup. But if the Finalize + // call is coming in the Destructor, we can't do any actual work in the + // process because that is likely to call "shared_from_this" which crashes + // if run while destructing. We use this flag to determine that. + std::atomic m_destructing; /// Mask for code an data addresses. The default value (0) means no mask is /// set. The bits set to 1 indicate bits that are NOT significant for diff --git a/lldb/include/lldb/Utility/Stream.h b/lldb/include/lldb/Utility/Stream.h index 1a5fd343e4df..20c55ac4597a 100644 --- a/lldb/include/lldb/Utility/Stream.h +++ b/lldb/include/lldb/Utility/Stream.h @@ -231,6 +231,40 @@ public: /// The string to be output to the stream. size_t PutCString(llvm::StringRef cstr); + /// Output a C string to the stream with color highlighting. + /// + /// Print a C string \a text to the stream, applying color highlighting to + /// the portions of the string that match the regex pattern \a pattern. The + /// pattern is matched as many times as possible throughout the string. If \a + /// pattern is nullptr, then no highlighting is applied. + /// + /// The highlighting is applied by enclosing the matching text in ANSI color + /// codes. The \a prefix parameter specifies the ANSI code to start the color + /// (the standard value is assumed to be 'ansi.fg.red', representing red + /// foreground), and the \a suffix parameter specifies the ANSI code to end + /// the color (the standard value is assumed to be 'ansi.normal', resetting to + /// default text style). These constants should be defined appropriately in + /// your environment. + /// + /// \param[in] text + /// The string to be output to the stream. + /// + /// \param[in] pattern + /// The regex pattern to match against the \a text string. Portions of \a + /// text matching this pattern will be colorized. If this parameter is + /// nullptr, highlighting is not performed. + /// \param[in] prefix + /// The ANSI color code to start colorization. This is + /// environment-dependent. + /// \param[in] suffix + /// The ANSI color code to end colorization. This is + /// environment-dependent. + + void PutCStringColorHighlighted(llvm::StringRef text, + llvm::StringRef pattern = "", + llvm::StringRef prefix = "", + llvm::StringRef suffix = ""); + /// Output and End of Line character to the stream. size_t EOL(); diff --git a/lldb/source/Commands/CommandCompletions.cpp b/lldb/source/Commands/CommandCompletions.cpp index 4d7e3d7f2497..0b69ce098195 100644 --- a/lldb/source/Commands/CommandCompletions.cpp +++ b/lldb/source/Commands/CommandCompletions.cpp @@ -333,7 +333,7 @@ static void DiskFilesOrDirectories(const llvm::Twine &partial_name, llvm::StringRef SearchDir; llvm::StringRef PartialItem; - if (CompletionBuffer.startswith("~")) { + if (CompletionBuffer.starts_with("~")) { llvm::StringRef Buffer = CompletionBuffer; size_t FirstSep = Buffer.find_if([](char c) { return path::is_separator(c); }); diff --git a/lldb/source/Commands/CommandObjectTarget.cpp b/lldb/source/Commands/CommandObjectTarget.cpp index 58785cde3ec7..63232c221ad1 100644 --- a/lldb/source/Commands/CommandObjectTarget.cpp +++ b/lldb/source/Commands/CommandObjectTarget.cpp @@ -8,6 +8,7 @@ #include "CommandObjectTarget.h" +#include "lldb/Core/Address.h" #include "lldb/Core/Debugger.h" #include "lldb/Core/IOHandler.h" #include "lldb/Core/Module.h" @@ -1532,7 +1533,7 @@ static void DumpOsoFilesTable(Stream &strm, static void DumpAddress(ExecutionContextScope *exe_scope, const Address &so_addr, bool verbose, bool all_ranges, - Stream &strm) { + Stream &strm, llvm::StringRef pattern = "") { strm.IndentMore(); strm.Indent(" Address: "); so_addr.Dump(&strm, exe_scope, Address::DumpStyleModuleWithFileAddress); @@ -1542,13 +1543,14 @@ static void DumpAddress(ExecutionContextScope *exe_scope, strm.Indent(" Summary: "); const uint32_t save_indent = strm.GetIndentLevel(); strm.SetIndentLevel(save_indent + 13); - so_addr.Dump(&strm, exe_scope, Address::DumpStyleResolvedDescription); + so_addr.Dump(&strm, exe_scope, Address::DumpStyleResolvedDescription, + Address::DumpStyleInvalid, UINT32_MAX, false, pattern); strm.SetIndentLevel(save_indent); // Print out detailed address information when verbose is enabled if (verbose) { strm.EOL(); so_addr.Dump(&strm, exe_scope, Address::DumpStyleDetailedSymbolContext, - Address::DumpStyleInvalid, UINT32_MAX, all_ranges); + Address::DumpStyleInvalid, UINT32_MAX, all_ranges, pattern); } strm.IndentLess(); } @@ -1593,6 +1595,7 @@ static uint32_t LookupSymbolInModule(CommandInterpreter &interpreter, return 0; SymbolContext sc; + const bool use_color = interpreter.GetDebugger().GetUseColor(); std::vector match_indexes; ConstString symbol_name(name); uint32_t num_matches = 0; @@ -1618,12 +1621,19 @@ static uint32_t LookupSymbolInModule(CommandInterpreter &interpreter, if (symbol->ValueIsAddress()) { DumpAddress( interpreter.GetExecutionContext().GetBestExecutionContextScope(), - symbol->GetAddressRef(), verbose, all_ranges, strm); + symbol->GetAddressRef(), verbose, all_ranges, strm, + use_color && name_is_regex ? name : nullptr); strm.EOL(); } else { strm.IndentMore(); strm.Indent(" Name: "); - strm.PutCString(symbol->GetDisplayName().GetStringRef()); + llvm::StringRef ansi_prefix = + interpreter.GetDebugger().GetRegexMatchAnsiPrefix(); + llvm::StringRef ansi_suffix = + interpreter.GetDebugger().GetRegexMatchAnsiSuffix(); + strm.PutCStringColorHighlighted( + symbol->GetDisplayName().GetStringRef(), + use_color ? name : nullptr, ansi_prefix, ansi_suffix); strm.EOL(); strm.Indent(" Value: "); strm.Printf("0x%16.16" PRIx64 "\n", symbol->GetRawValue()); diff --git a/lldb/source/Core/Address.cpp b/lldb/source/Core/Address.cpp index 189d50fe962a..19d34db44ea5 100644 --- a/lldb/source/Core/Address.cpp +++ b/lldb/source/Core/Address.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "lldb/Core/Address.h" +#include "lldb/Core/Debugger.h" #include "lldb/Core/Declaration.h" #include "lldb/Core/DumpDataExtractor.h" #include "lldb/Core/Module.h" @@ -28,6 +29,7 @@ #include "lldb/Target/Process.h" #include "lldb/Target/SectionLoadList.h" #include "lldb/Target/Target.h" +#include "lldb/Utility/AnsiTerminal.h" #include "lldb/Utility/ConstString.h" #include "lldb/Utility/DataExtractor.h" #include "lldb/Utility/Endian.h" @@ -405,7 +407,7 @@ bool Address::GetDescription(Stream &s, Target &target, bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, DumpStyle fallback_style, uint32_t addr_size, - bool all_ranges) const { + bool all_ranges, llvm::StringRef pattern) const { // If the section was nullptr, only load address is going to work unless we // are trying to deref a pointer SectionSP section_sp(GetSection()); @@ -501,7 +503,6 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, pointer_size = target->GetArchitecture().GetAddressByteSize(); else if (module_sp) pointer_size = module_sp->GetArchitecture().GetAddressByteSize(); - bool showed_info = false; if (section_sp) { SectionType sect_type = section_sp->GetType(); @@ -515,7 +516,16 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, if (symbol) { const char *symbol_name = symbol->GetName().AsCString(); if (symbol_name) { - s->PutCString(symbol_name); + llvm::StringRef ansi_prefix; + llvm::StringRef ansi_suffix; + if (target) { + ansi_prefix = + target->GetDebugger().GetRegexMatchAnsiPrefix(); + ansi_suffix = + target->GetDebugger().GetRegexMatchAnsiSuffix(); + } + s->PutCStringColorHighlighted(symbol_name, pattern, + ansi_prefix, ansi_suffix); addr_t delta = file_Addr - symbol->GetAddressRef().GetFileAddress(); if (delta) @@ -643,7 +653,7 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, pointer_sc.symbol != nullptr) { s->PutCString(": "); pointer_sc.DumpStopContext(s, exe_scope, so_addr, true, false, - false, true, true); + false, true, true, pattern); } } } @@ -682,19 +692,22 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, // address. sc.DumpStopContext(s, exe_scope, *this, show_fullpaths, show_module, show_inlined_frames, - show_function_arguments, show_function_name); + show_function_arguments, show_function_name, + pattern); } else { // We found a symbol but it was in a different section so it // isn't the symbol we should be showing, just show the section // name + offset - Dump(s, exe_scope, DumpStyleSectionNameOffset); + Dump(s, exe_scope, DumpStyleSectionNameOffset, DumpStyleInvalid, + UINT32_MAX, false, pattern); } } } } } else { if (fallback_style != DumpStyleInvalid) - return Dump(s, exe_scope, fallback_style, DumpStyleInvalid, addr_size); + return Dump(s, exe_scope, fallback_style, DumpStyleInvalid, addr_size, + false, pattern); return false; } break; @@ -715,7 +728,7 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, sc.symbol->GetAddressRef().GetSection() != GetSection()) sc.symbol = nullptr; } - sc.GetDescription(s, eDescriptionLevelBrief, target); + sc.GetDescription(s, eDescriptionLevelBrief, target, pattern); if (sc.block) { bool can_create = true; @@ -763,7 +776,8 @@ bool Address::Dump(Stream *s, ExecutionContextScope *exe_scope, DumpStyle style, } } else { if (fallback_style != DumpStyleInvalid) - return Dump(s, exe_scope, fallback_style, DumpStyleInvalid, addr_size); + return Dump(s, exe_scope, fallback_style, DumpStyleInvalid, addr_size, + false, pattern); return false; } break; diff --git a/lldb/source/Core/CoreProperties.td b/lldb/source/Core/CoreProperties.td index 0e0f468d3ecd..8d81967bdb50 100644 --- a/lldb/source/Core/CoreProperties.td +++ b/lldb/source/Core/CoreProperties.td @@ -203,6 +203,14 @@ let Definition = "debugger" in { Global, DefaultStringValue<"${ansi.normal}">, Desc<"When displaying suggestion in a color-enabled terminal, use the ANSI terminal code specified in this format immediately after the suggestion.">; + def ShowRegexMatchAnsiPrefix: Property<"show-regex-match-ansi-prefix", "String">, + Global, + DefaultStringValue<"${ansi.fg.red}">, + Desc<"When displaying a regex match in a color-enabled terminal, use the ANSI terminal code specified in this format immediately before the match.">; + def ShowRegexMatchAnsiSuffix: Property<"show-regex-match-ansi-suffix", "String">, + Global, + DefaultStringValue<"${ansi.normal}">, + Desc<"When displaying a regex match in a color-enabled terminal, use the ANSI terminal code specified in this format immediately after the match.">; def ShowDontUsePoHint: Property<"show-dont-use-po-hint", "Boolean">, Global, DefaultTrue, diff --git a/lldb/source/Core/Debugger.cpp b/lldb/source/Core/Debugger.cpp index 21f71e449ca5..97311b4716ac 100644 --- a/lldb/source/Core/Debugger.cpp +++ b/lldb/source/Core/Debugger.cpp @@ -453,6 +453,18 @@ llvm::StringRef Debugger::GetAutosuggestionAnsiSuffix() const { idx, g_debugger_properties[idx].default_cstr_value); } +llvm::StringRef Debugger::GetRegexMatchAnsiPrefix() const { + const uint32_t idx = ePropertyShowRegexMatchAnsiPrefix; + return GetPropertyAtIndexAs( + idx, g_debugger_properties[idx].default_cstr_value); +} + +llvm::StringRef Debugger::GetRegexMatchAnsiSuffix() const { + const uint32_t idx = ePropertyShowRegexMatchAnsiSuffix; + return GetPropertyAtIndexAs( + idx, g_debugger_properties[idx].default_cstr_value); +} + bool Debugger::GetShowDontUsePoHint() const { const uint32_t idx = ePropertyShowDontUsePoHint; return GetPropertyAtIndexAs( @@ -930,7 +942,7 @@ void Debugger::Clear() { for (TargetSP target_sp : m_target_list.Targets()) { if (target_sp) { if (ProcessSP process_sp = target_sp->GetProcessSP()) - process_sp->Finalize(); + process_sp->Finalize(false /* not destructing */); target_sp->Destroy(); } } diff --git a/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp b/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp index 70bb9aa7a833..de739acf5b2a 100644 --- a/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp +++ b/lldb/source/Plugins/Process/MacOSX-Kernel/ProcessKDP.cpp @@ -164,7 +164,7 @@ ProcessKDP::~ProcessKDP() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); } Status ProcessKDP::DoWillLaunch(Module *module) { diff --git a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp index aedc43a015ff..a4540de4acc4 100644 --- a/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp +++ b/lldb/source/Plugins/Process/elf-core/ProcessElfCore.cpp @@ -108,7 +108,7 @@ ProcessElfCore::~ProcessElfCore() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); } lldb::addr_t ProcessElfCore::AddAddressRangeFromLoadSegment( diff --git a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp index b04319703b94..d5e557b4b88c 100644 --- a/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp +++ b/lldb/source/Plugins/Process/gdb-remote/ProcessGDBRemote.cpp @@ -303,7 +303,7 @@ ProcessGDBRemote::~ProcessGDBRemote() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); // The general Finalize is going to try to destroy the process and that // SHOULD shut down the async thread. However, if we don't kill it it will diff --git a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp index 9830a4b8599d..a2ea19388b75 100644 --- a/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp +++ b/lldb/source/Plugins/Process/mach-core/ProcessMachCore.cpp @@ -123,7 +123,7 @@ ProcessMachCore::~ProcessMachCore() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); } bool ProcessMachCore::CheckAddressForDyldOrKernel(lldb::addr_t addr, diff --git a/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp b/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp index 0d5ca42691d3..b72307c7e4b9 100644 --- a/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp +++ b/lldb/source/Plugins/Process/minidump/ProcessMinidump.cpp @@ -166,7 +166,7 @@ ProcessMinidump::~ProcessMinidump() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); } void ProcessMinidump::Initialize() { diff --git a/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp b/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp index 54b367727913..66f861350d14 100644 --- a/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp +++ b/lldb/source/Plugins/Process/scripted/ScriptedProcess.cpp @@ -140,7 +140,7 @@ ScriptedProcess::~ScriptedProcess() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); } void ScriptedProcess::Initialize() { diff --git a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp index 44e760227901..44421c0eda3e 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/DWARFDeclContext.cpp @@ -7,10 +7,27 @@ //===----------------------------------------------------------------------===// #include "DWARFDeclContext.h" +#include "llvm/Support/raw_ostream.h" using namespace lldb_private::dwarf; using namespace lldb_private::plugin::dwarf; +/// Returns the name of `entry` if it has one, or the appropriate "anonymous +/// {namespace, class, struct, union}". +static const char *GetName(DWARFDeclContext::Entry entry) { + if (entry.name != nullptr) + return entry.name; + if (entry.tag == DW_TAG_namespace) + return "(anonymous namespace)"; + if (entry.tag == DW_TAG_class_type) + return "(anonymous class)"; + if (entry.tag == DW_TAG_structure_type) + return "(anonymous struct)"; + if (entry.tag == DW_TAG_union_type) + return "(anonymous union)"; + return "(anonymous)"; +} + const char *DWARFDeclContext::GetQualifiedName() const { if (m_qualified_name.empty()) { // The declaration context array for a class named "foo" in namespace @@ -26,26 +43,10 @@ const char *DWARFDeclContext::GetQualifiedName() const { m_qualified_name.append(m_entries[0].name); } } else { - collection::const_reverse_iterator pos; - collection::const_reverse_iterator begin = m_entries.rbegin(); - collection::const_reverse_iterator end = m_entries.rend(); - for (pos = begin; pos != end; ++pos) { - if (pos != begin) - m_qualified_name.append("::"); - if (pos->name == nullptr) { - if (pos->tag == DW_TAG_namespace) - m_qualified_name.append("(anonymous namespace)"); - else if (pos->tag == DW_TAG_class_type) - m_qualified_name.append("(anonymous class)"); - else if (pos->tag == DW_TAG_structure_type) - m_qualified_name.append("(anonymous struct)"); - else if (pos->tag == DW_TAG_union_type) - m_qualified_name.append("(anonymous union)"); - else - m_qualified_name.append("(anonymous)"); - } else - m_qualified_name.append(pos->name); - } + llvm::raw_string_ostream string_stream(m_qualified_name); + llvm::interleave( + llvm::reverse(m_entries), string_stream, + [&](auto entry) { string_stream << GetName(entry); }, "::"); } } } diff --git a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp index d4cc26a3c329..d4c573ecd468 100644 --- a/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp +++ b/lldb/source/Plugins/SymbolFile/DWARF/SymbolFileDWARF.cpp @@ -132,6 +132,11 @@ public: } // namespace +bool IsStructOrClassTag(llvm::dwarf::Tag Tag) { + return Tag == llvm::dwarf::Tag::DW_TAG_class_type || + Tag == llvm::dwarf::Tag::DW_TAG_structure_type; +} + static PluginProperties &GetGlobalPluginProperties() { static PluginProperties g_settings; return g_settings; @@ -2947,29 +2952,18 @@ TypeSP SymbolFileDWARF::FindCompleteObjCDefinitionTypeForDIE( m_index->GetCompleteObjCClass( type_name, must_be_implementation, [&](DWARFDIE type_die) { - bool try_resolving_type = false; - // Don't try and resolve the DIE we are looking for with the DIE // itself! - if (type_die != die) { - switch (type_die.Tag()) { - case DW_TAG_class_type: - case DW_TAG_structure_type: - try_resolving_type = true; - break; - default: - break; - } - } - if (!try_resolving_type) + if (type_die == die || !IsStructOrClassTag(type_die.Tag())) return true; if (must_be_implementation && - type_die.Supports_DW_AT_APPLE_objc_complete_type()) - try_resolving_type = type_die.GetAttributeValueAsUnsigned( + type_die.Supports_DW_AT_APPLE_objc_complete_type()) { + const bool try_resolving_type = type_die.GetAttributeValueAsUnsigned( DW_AT_APPLE_objc_complete_type, 0); - if (!try_resolving_type) - return true; + if (!try_resolving_type) + return true; + } Type *resolved_type = ResolveType(type_die, false, true); if (!resolved_type || resolved_type == DIE_IS_BEING_PARSED) @@ -3128,36 +3122,12 @@ SymbolFileDWARF::FindDefinitionTypeForDWARFDeclContext(const DWARFDIE &die) { if (type_system && !type_system->SupportsLanguage(GetLanguage(*type_die.GetCU()))) return true; - bool try_resolving_type = false; - // Don't try and resolve the DIE we are looking for with the DIE - // itself! const dw_tag_t type_tag = type_die.Tag(); - // Make sure the tags match - if (type_tag == tag) { - // The tags match, lets try resolving this type - try_resolving_type = true; - } else { - // The tags don't match, but we need to watch our for a forward - // declaration for a struct and ("struct foo") ends up being a - // class ("class foo { ... };") or vice versa. - switch (type_tag) { - case DW_TAG_class_type: - // We had a "class foo", see if we ended up with a "struct foo - // { ... };" - try_resolving_type = (tag == DW_TAG_structure_type); - break; - case DW_TAG_structure_type: - // We had a "struct foo", see if we ended up with a "class foo - // { ... };" - try_resolving_type = (tag == DW_TAG_class_type); - break; - default: - // Tags don't match, don't event try to resolve using this type - // whose name matches.... - break; - } - } + // Resolve the type if both have the same tag or {class, struct} tags. + const bool try_resolving_type = + type_tag == tag || + (IsStructOrClassTag(type_tag) && IsStructOrClassTag(tag)); if (!try_resolving_type) { if (log) { diff --git a/lldb/source/Symbol/Symbol.cpp b/lldb/source/Symbol/Symbol.cpp index 26b4c4d62ad9..fcc45f861c22 100644 --- a/lldb/source/Symbol/Symbol.cpp +++ b/lldb/source/Symbol/Symbol.cpp @@ -8,6 +8,8 @@ #include "lldb/Symbol/Symbol.h" +#include "lldb/Core/Address.h" +#include "lldb/Core/Debugger.h" #include "lldb/Core/Module.h" #include "lldb/Core/ModuleSpec.h" #include "lldb/Core/Section.h" @@ -225,7 +227,7 @@ bool Symbol::IsTrampoline() const { return m_type == eSymbolTypeTrampoline; } bool Symbol::IsIndirect() const { return m_type == eSymbolTypeResolver; } void Symbol::GetDescription(Stream *s, lldb::DescriptionLevel level, - Target *target) const { + Target *target, llvm::StringRef pattern) const { s->Printf("id = {0x%8.8x}", m_uid); if (m_addr_range.GetBaseAddress().GetSection()) { @@ -252,11 +254,24 @@ void Symbol::GetDescription(Stream *s, lldb::DescriptionLevel level, s->Printf(", value = 0x%16.16" PRIx64, m_addr_range.GetBaseAddress().GetOffset()); } - ConstString demangled = GetMangled().GetDemangledName(); - if (demangled) - s->Printf(", name=\"%s\"", demangled.AsCString()); - if (m_mangled.GetMangledName()) - s->Printf(", mangled=\"%s\"", m_mangled.GetMangledName().AsCString()); + llvm::StringRef ansi_prefix; + llvm::StringRef ansi_suffix; + if (target) { + ansi_prefix = target->GetDebugger().GetRegexMatchAnsiPrefix(); + ansi_suffix = target->GetDebugger().GetRegexMatchAnsiSuffix(); + } + if (ConstString demangled = m_mangled.GetDemangledName()) { + s->PutCString(", name=\""); + s->PutCStringColorHighlighted(demangled.GetStringRef(), pattern, + ansi_prefix, ansi_suffix); + s->PutCString("\""); + } + if (ConstString mangled_name = m_mangled.GetMangledName()) { + s->PutCString(", mangled=\""); + s->PutCStringColorHighlighted(mangled_name.GetStringRef(), pattern, + ansi_prefix, ansi_suffix); + s->PutCString("\""); + } } void Symbol::Dump(Stream *s, Target *target, uint32_t index, diff --git a/lldb/source/Symbol/SymbolContext.cpp b/lldb/source/Symbol/SymbolContext.cpp index 63968ec2d150..9fd40b5ca567 100644 --- a/lldb/source/Symbol/SymbolContext.cpp +++ b/lldb/source/Symbol/SymbolContext.cpp @@ -8,6 +8,7 @@ #include "lldb/Symbol/SymbolContext.h" +#include "lldb/Core/Address.h" #include "lldb/Core/Debugger.h" #include "lldb/Core/Module.h" #include "lldb/Core/ModuleSpec.h" @@ -71,7 +72,8 @@ bool SymbolContext::DumpStopContext(Stream *s, ExecutionContextScope *exe_scope, const Address &addr, bool show_fullpaths, bool show_module, bool show_inlined_frames, bool show_function_arguments, - bool show_function_name) const { + bool show_function_name, + llvm::StringRef pattern) const { bool dumped_something = false; if (show_module && module_sp) { if (show_fullpaths) @@ -81,7 +83,6 @@ bool SymbolContext::DumpStopContext(Stream *s, ExecutionContextScope *exe_scope, s->PutChar('`'); dumped_something = true; } - if (function != nullptr) { SymbolContext inline_parent_sc; Address inline_parent_addr; @@ -94,8 +95,16 @@ bool SymbolContext::DumpStopContext(Stream *s, ExecutionContextScope *exe_scope, name = function->GetNameNoArguments(); if (!name) name = function->GetName(); - if (name) - name.Dump(s); + if (name) { + llvm::StringRef ansi_prefix; + llvm::StringRef ansi_suffix; + if (target_sp) { + ansi_prefix = target_sp->GetDebugger().GetRegexMatchAnsiPrefix(); + ansi_suffix = target_sp->GetDebugger().GetRegexMatchAnsiSuffix(); + } + s->PutCStringColorHighlighted(name.GetStringRef(), pattern, ansi_prefix, + ansi_suffix); + } } if (addr.IsValid()) { @@ -163,7 +172,14 @@ bool SymbolContext::DumpStopContext(Stream *s, ExecutionContextScope *exe_scope, dumped_something = true; if (symbol->GetType() == eSymbolTypeTrampoline) s->PutCString("symbol stub for: "); - symbol->GetName().Dump(s); + llvm::StringRef ansi_prefix; + llvm::StringRef ansi_suffix; + if (target_sp) { + ansi_prefix = target_sp->GetDebugger().GetRegexMatchAnsiPrefix(); + ansi_suffix = target_sp->GetDebugger().GetRegexMatchAnsiSuffix(); + } + s->PutCStringColorHighlighted(symbol->GetName().GetStringRef(), pattern, + ansi_prefix, ansi_suffix); } if (addr.IsValid() && symbol->ValueIsAddress()) { @@ -186,7 +202,8 @@ bool SymbolContext::DumpStopContext(Stream *s, ExecutionContextScope *exe_scope, } void SymbolContext::GetDescription(Stream *s, lldb::DescriptionLevel level, - Target *target) const { + Target *target, + llvm::StringRef pattern) const { if (module_sp) { s->Indent(" Module: file = \""); module_sp->GetFileSpec().Dump(s->AsRawOstream()); @@ -246,7 +263,7 @@ void SymbolContext::GetDescription(Stream *s, lldb::DescriptionLevel level, if (symbol != nullptr) { s->Indent(" Symbol: "); - symbol->GetDescription(s, level, target); + symbol->GetDescription(s, level, target, pattern); s->EOL(); } diff --git a/lldb/source/Symbol/Variable.cpp b/lldb/source/Symbol/Variable.cpp index 85ceadd20c61..db740cb7cb6e 100644 --- a/lldb/source/Symbol/Variable.cpp +++ b/lldb/source/Symbol/Variable.cpp @@ -227,7 +227,8 @@ bool Variable::LocationIsValidForFrame(StackFrame *frame) { // contains the current address when converted to a load address return m_location_list.ContainsAddress( loclist_base_load_addr, - frame->GetFrameCodeAddress().GetLoadAddress(target_sp.get())); + frame->GetFrameCodeAddressForSymbolication().GetLoadAddress( + target_sp.get())); } } return false; diff --git a/lldb/source/Target/Process.cpp b/lldb/source/Target/Process.cpp index 2d77144a9b28..aa3b04c43cc5 100644 --- a/lldb/source/Target/Process.cpp +++ b/lldb/source/Target/Process.cpp @@ -445,7 +445,7 @@ Process::Process(lldb::TargetSP target_sp, ListenerSP listener_sp, m_memory_cache(*this), m_allocated_memory_cache(*this), m_should_detach(false), m_next_event_action_up(), m_public_run_lock(), m_private_run_lock(), m_currently_handling_do_on_removals(false), - m_resume_requested(false), m_finalizing(false), + m_resume_requested(false), m_finalizing(false), m_destructing(false), m_clear_thread_plans_on_stop(false), m_force_next_event_delivery(false), m_last_broadcast_state(eStateInvalid), m_destroy_in_process(false), m_can_interpret_function_calls(false), m_run_thread_plan_lock(), @@ -518,9 +518,11 @@ ProcessProperties &Process::GetGlobalProperties() { return *g_settings_ptr; } -void Process::Finalize() { +void Process::Finalize(bool destructing) { if (m_finalizing.exchange(true)) return; + if (destructing) + m_destructing.exchange(true); // Destroy the process. This will call the virtual function DoDestroy under // the hood, giving our derived class a chance to do the ncessary tear down. @@ -1415,7 +1417,13 @@ bool Process::StateChangedIsHijackedForSynchronousResume() { StateType Process::GetPrivateState() { return m_private_state.GetValue(); } void Process::SetPrivateState(StateType new_state) { - if (m_finalizing) + // Use m_destructing not m_finalizing here. If we are finalizing a process + // that we haven't started tearing down, we'd like to be able to nicely + // detach if asked, but that requires the event system be live. That will + // not be true for an in-the-middle-of-being-destructed Process, since the + // event system relies on Process::shared_from_this, which may have already + // been destroyed. + if (m_destructing) return; Log *log(GetLog(LLDBLog::State | LLDBLog::Process | LLDBLog::Unwind)); diff --git a/lldb/source/Target/ProcessTrace.cpp b/lldb/source/Target/ProcessTrace.cpp index 061af9e0e520..6e5ef6a379f9 100644 --- a/lldb/source/Target/ProcessTrace.cpp +++ b/lldb/source/Target/ProcessTrace.cpp @@ -50,7 +50,7 @@ ProcessTrace::~ProcessTrace() { // make sure all of the broadcaster cleanup goes as planned. If we destruct // this class, then Process::~Process() might have problems trying to fully // destroy the broadcaster. - Finalize(); + Finalize(true /* destructing */); } void ProcessTrace::DidAttach(ArchSpec &process_arch) { diff --git a/lldb/source/Target/Target.cpp b/lldb/source/Target/Target.cpp index 2e8d1dfdaa17..302c2bad7021 100644 --- a/lldb/source/Target/Target.cpp +++ b/lldb/source/Target/Target.cpp @@ -197,7 +197,7 @@ void Target::DeleteCurrentProcess() { if (m_process_sp->IsAlive()) m_process_sp->Destroy(false); - m_process_sp->Finalize(); + m_process_sp->Finalize(false /* not destructing */); CleanupProcess(); diff --git a/lldb/source/Utility/Stream.cpp b/lldb/source/Utility/Stream.cpp index af28a49a1f0c..62e061e9d09c 100644 --- a/lldb/source/Utility/Stream.cpp +++ b/lldb/source/Utility/Stream.cpp @@ -8,11 +8,13 @@ #include "lldb/Utility/Stream.h" +#include "lldb/Utility/AnsiTerminal.h" #include "lldb/Utility/Endian.h" #include "lldb/Utility/VASPrintf.h" #include "llvm/ADT/SmallString.h" #include "llvm/Support/Format.h" #include "llvm/Support/LEB128.h" +#include "llvm/Support/Regex.h" #include @@ -70,6 +72,34 @@ size_t Stream::PutCString(llvm::StringRef str) { return bytes_written; } +void Stream::PutCStringColorHighlighted(llvm::StringRef text, + llvm::StringRef pattern, + llvm::StringRef prefix, + llvm::StringRef suffix) { + // Only apply color formatting when a pattern is present and both prefix and + // suffix are specified. In the absence of these conditions, output the text + // without color formatting. + if (pattern.empty() || (prefix.empty() && suffix.empty())) { + PutCString(text); + return; + } + + llvm::Regex reg_pattern(pattern); + llvm::SmallVector matches; + llvm::StringRef remaining = text; + std::string format_str = lldb_private::ansi::FormatAnsiTerminalCodes( + prefix.str() + "%.*s" + suffix.str()); + while (reg_pattern.match(remaining, &matches)) { + llvm::StringRef match = matches[0]; + size_t match_start_pos = match.data() - remaining.data(); + PutCString(remaining.take_front(match_start_pos)); + Printf(format_str.c_str(), match.size(), match.data()); + remaining = remaining.drop_front(match_start_pos + match.size()); + } + if (remaining.size()) + PutCString(remaining); +} + // Print a double quoted NULL terminated C string to the stream using the // printf format in "format". void Stream::QuotedCString(const char *cstr, const char *format) { diff --git a/lldb/test/API/driver/quit_speed/Makefile b/lldb/test/API/driver/quit_speed/Makefile new file mode 100644 index 000000000000..10495940055b --- /dev/null +++ b/lldb/test/API/driver/quit_speed/Makefile @@ -0,0 +1,3 @@ +C_SOURCES := main.c + +include Makefile.rules diff --git a/lldb/test/API/driver/quit_speed/TestQuitWithProcess.py b/lldb/test/API/driver/quit_speed/TestQuitWithProcess.py new file mode 100644 index 000000000000..957586d41f6b --- /dev/null +++ b/lldb/test/API/driver/quit_speed/TestQuitWithProcess.py @@ -0,0 +1,34 @@ +""" +Test that killing the target while quitting doesn't stall +""" + + +import lldb +from lldbsuite.test.decorators import * +from lldbsuite.test.lldbtest import * +from lldbsuite.test import lldbutil +import pexpect +from lldbsuite.test.lldbpexpect import PExpectTest + + +class DriverQuitSpeedTest(PExpectTest): + source = "main.c" + + def test_run_quit(self): + """Test that the lldb driver's batch mode works correctly.""" + self.build() + + exe = self.getBuildArtifact("a.out") + + # Turn on auto-confirm removes the wait for the prompt. + self.launch(executable=exe, extra_args=["-O", "settings set auto-confirm 1"]) + child = self.child + + # Launch the process without a TTY so we don't have to interrupt: + child.sendline("process launch -n") + print("launched process") + child.expect("Process ([\d]*) launched:") + print("Got launch message") + child.sendline("quit") + print("sent quit") + child.expect(pexpect.EOF, timeout=15) diff --git a/lldb/test/API/driver/quit_speed/main.c b/lldb/test/API/driver/quit_speed/main.c new file mode 100644 index 000000000000..3d6d45ce5e80 --- /dev/null +++ b/lldb/test/API/driver/quit_speed/main.c @@ -0,0 +1,8 @@ +#include + +int main (int argc, char **argv) { + while(1) + usleep(5); + + return 0; +} diff --git a/lldb/test/API/functionalities/location-list-lookup/Makefile b/lldb/test/API/functionalities/location-list-lookup/Makefile index 78b0b11cb748..8e453681d7b3 100644 --- a/lldb/test/API/functionalities/location-list-lookup/Makefile +++ b/lldb/test/API/functionalities/location-list-lookup/Makefile @@ -1,3 +1,3 @@ -C_SOURCES := main.c +CXX_SOURCES := main.cpp CFLAGS_EXTRAS := -O1 include Makefile.rules diff --git a/lldb/test/API/functionalities/location-list-lookup/TestLocationListLookup.py b/lldb/test/API/functionalities/location-list-lookup/TestLocationListLookup.py index 4793447c5941..feea14ff355e 100644 --- a/lldb/test/API/functionalities/location-list-lookup/TestLocationListLookup.py +++ b/lldb/test/API/functionalities/location-list-lookup/TestLocationListLookup.py @@ -7,16 +7,8 @@ from lldbsuite.test import lldbutil class LocationListLookupTestCase(TestBase): - def setUp(self): - # Call super's setUp(). - TestBase.setUp(self) - - @skipIf(oslist=["linux"], archs=["arm"]) - def test_loclist(self): - self.build() + def launch(self) -> lldb.SBProcess: exe = self.getBuildArtifact("a.out") - - # Create a target by the debugger. target = self.dbg.CreateTarget(exe) self.assertTrue(target, VALID_TARGET) self.dbg.SetAsync(False) @@ -27,12 +19,32 @@ class LocationListLookupTestCase(TestBase): self.assertTrue(process.IsValid()) self.assertTrue(process.is_stopped) - # Find `main` on the stack, then - # find `argv` local variable, then - # check that we can read the c-string in argv[0] + return process + + def check_local_vars(self, process: lldb.SBProcess, check_expr: bool): + # Find `bar` on the stack, then + # make sure we can read out the local + # variables (with both `frame var` and `expr`) for f in process.GetSelectedThread().frames: - if f.GetDisplayFunctionName() == "main": + frame_name = f.GetDisplayFunctionName() + if frame_name is not None and frame_name.startswith("Foo::bar"): argv = f.GetValueForVariablePath("argv").GetChildAtIndex(0) strm = lldb.SBStream() argv.GetDescription(strm) self.assertNotEqual(strm.GetData().find("a.out"), -1) + + if check_expr: + process.GetSelectedThread().SetSelectedFrame(f.idx) + self.expect_expr("this", result_type="Foo *") + + @skipIf(oslist=["linux"], archs=["arm"]) + @skipIfDarwin + def test_loclist_frame_var(self): + self.build() + self.check_local_vars(self.launch(), check_expr=False) + + @skipIf(compiler="clang", compiler_version=["<=", "11.0"]) + @skipUnlessDarwin + def test_loclist_expr(self): + self.build() + self.check_local_vars(self.launch(), check_expr=True) diff --git a/lldb/test/API/functionalities/location-list-lookup/main.c b/lldb/test/API/functionalities/location-list-lookup/main.c deleted file mode 100644 index 852772ee52ca..000000000000 --- a/lldb/test/API/functionalities/location-list-lookup/main.c +++ /dev/null @@ -1,23 +0,0 @@ -#include -#include - -// The goal with this test is: -// 1. Have main() followed by foo() -// 2. Have the no-return call to abort() in main be the last instruction -// 3. Have the next instruction be the start of foo() -// 4. The debug info for argv uses a location list. -// clang at -O1 on x86_64 or arm64 has debuginfo like -// DW_AT_location (0x00000049: -// [0x0000000100003f15, 0x0000000100003f25): DW_OP_reg4 RSI -// [0x0000000100003f25, 0x0000000100003f5b): DW_OP_reg15 R15) - -void foo(int); -int main(int argc, char **argv) { - char *file = argv[0]; - char f0 = file[0]; - printf("%c\n", f0); - foo(f0); - printf("%s %d\n", argv[0], argc); - abort(); /// argv is still be accessible here -} -void foo(int in) { printf("%d\n", in); } diff --git a/lldb/test/API/functionalities/location-list-lookup/main.cpp b/lldb/test/API/functionalities/location-list-lookup/main.cpp new file mode 100644 index 000000000000..4ccdadbddbb5 --- /dev/null +++ b/lldb/test/API/functionalities/location-list-lookup/main.cpp @@ -0,0 +1,23 @@ +#include +#include + +void func(int in); + +struct Foo { + int x; + [[clang::noinline]] void bar(char **argv); +}; + +int main(int argc, char **argv) { + Foo f{.x = 5}; + std::printf("%p\n", &f.x); + f.bar(argv); + return f.x; +} + +void Foo::bar(char **argv) { + std::printf("%p %p\n", argv, this); + std::abort(); /// 'this' should be still accessible +} + +void func(int in) { printf("%d\n", in); } diff --git a/lldb/test/API/lang/cpp/const_static_integral_member/TestConstStaticIntegralMember.py b/lldb/test/API/lang/cpp/const_static_integral_member/TestConstStaticIntegralMember.py index 60e116b42207..cdade2d335df 100644 --- a/lldb/test/API/lang/cpp/const_static_integral_member/TestConstStaticIntegralMember.py +++ b/lldb/test/API/lang/cpp/const_static_integral_member/TestConstStaticIntegralMember.py @@ -104,7 +104,7 @@ class TestCase(TestBase): # dsymutil strips the debug info for classes that only have const static # data members without locations. - @expectedFailureAll(debug_info=["dsym"]) + @expectedFailureAll(debug_info=["dsym"], dwarf_version=["<", "5"]) def test_class_with_only_const_static(self): self.build() lldbutil.run_to_source_breakpoint( @@ -120,6 +120,9 @@ class TestCase(TestBase): self.assertEqual(varobj.type.name, expect_type) self.assertEqual(varobj.value, expect_val) + @expectedFailureAll(dwarf_version=["<", "5"]) + # On linux this passes due to the manual index + @expectedFailureDarwin(debug_info=no_match(["dsym"])) def test_inline_static_members(self): self.build() lldbutil.run_to_source_breakpoint( @@ -167,6 +170,9 @@ class TestCase(TestBase): "ClassWithEnumAlias::enum_alias_alias", result_value="scoped_enum_case1" ) + @expectedFailureAll(dwarf_version=["<", "5"]) + # On linux this passes due to the manual index + @expectedFailureDarwin(debug_info=no_match(["dsym"])) def test_shadowed_static_inline_members(self): """Tests that the expression evaluator and SBAPI can both correctly determine the requested inline static variable diff --git a/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py b/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py index d2a21ad3cd1d..9b0755eea7d3 100644 --- a/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py +++ b/lldb/test/API/tools/lldb-dap/variables/TestDAP_variables.py @@ -4,8 +4,8 @@ Test lldb-dap setBreakpoints request import os -import lldbdap_testcase import dap_server +import lldbdap_testcase from lldbsuite.test import lldbutil from lldbsuite.test.decorators import * from lldbsuite.test.lldbtest import * @@ -152,7 +152,13 @@ class TestDAP_variables(lldbdap_testcase.DAPTestCaseBase): globals = self.dap_server.get_global_variables() buffer_children = make_buffer_verify_dict(0, 32) verify_locals = { - "argc": {"equals": {"type": "int", "value": "1"}}, + "argc": { + "equals": {"type": "int", "value": "1"}, + "declaration": { + "equals": {"line": 12, "column": 14}, + "contains": {"path": ["lldb-dap", "variables", "main.cpp"]}, + }, + }, "argv": { "equals": {"type": "const char **"}, "startswith": {"value": "0x"}, diff --git a/lldb/test/Shell/Commands/command-image-lookup-color.test b/lldb/test/Shell/Commands/command-image-lookup-color.test new file mode 100644 index 000000000000..186526b7efae --- /dev/null +++ b/lldb/test/Shell/Commands/command-image-lookup-color.test @@ -0,0 +1,60 @@ +# RUN: %clang_host -g %S/Inputs/main.c -o %t + +# The file main.c is does not show up in search on Windows. +# UNSUPPORTED: system-windows + +# Until we figure out the correct CHECK lines. +# UNSUPPORTED: system-darwin + +# Checking simple regex search + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'image lookup -r -s ma' | FileCheck %s --check-prefix CHECK1 +# CHECK1: Name: {{.+}}31mma{{.+}}0min.c + +# Checking complex regex searches + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'image lookup -r -s main.c|foo' | FileCheck %s --check-prefix CHECK2 +# CHECK2: Name: {{.+}}31mmain.c{{.+}}0m + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'image lookup -r -s m[abc]' | FileCheck %s --check-prefix CHECK3 +# CHECK3: Name: {{.+}}31mma{{.+}}0min.c + +# Checking to ensure that no attempt is made to color anything when there are no matching symbols found + +# RUN: %lldb %t -o 'settings set use-color true' -o 'image lookup -r -s IMPPATTERN123456' | FileCheck %s --check-prefix CHECK4 +# CHECK4-NOT: {{[0-9]+}} symbols match the regular expression + +# Checking multiple matches on same symbol + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'image lookup -r -s (ma|n)' | FileCheck %s --check-prefix CHECK5 +# CHECK5: Name: {{.+}}31mma{{.+}}0mi{{.+}}31mn{{.+}}0m.c + +# Checking no colorization without regex search + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'image lookup -s main' | FileCheck %s --check-prefix CHECK6 +# CHECK6: Summary: {{.+}}`main at main.c: + +# Checking no colorization when use-color is false + +# RUN: %lldb %t -b -o 'settings set use-color false' -o 'image lookup -r -s ma' | FileCheck %s --check-prefix CHECK7 +# CHECK7: Name: main.c + +# Checking for custom colors + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'settings set show-regex-match-ansi-prefix ${ansi.fg.green}' -o 'image lookup -r -s ma' | FileCheck %s --check-prefix CHECK8 +# CHECK8: Name: {{.+}}32mma{{.+}}0min.c + +# Checking for functionality when there's prefix but no suffix + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'settings set show-regex-match-ansi-prefix ${ansi.fg.red}' -o 'settings set show-regex-match-ansi-suffix ""' -o 'image lookup -r -s ma' | FileCheck %s --check-prefix CHECK9 +# CHECK9: Name: {{.+}}31mmain.c + +# Checking for functionality when there's suffix but no prefix + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'settings set show-regex-match-ansi-prefix ""' -o 'settings set show-regex-match-ansi-suffix ${ansi.fg.red}' -o 'image lookup -r -s ma' | FileCheck %s --check-prefix CHECK10 +# CHECK10: Name: ma{{.+}}31min.c + +# Checking for no colorization when there's neither suffix nor prefix + +# RUN: %lldb %t -b -o 'settings set use-color true' -o 'settings set show-regex-match-ansi-prefix ""' -o 'settings set show-regex-match-ansi-suffix ""' -o 'image lookup -r -s ma' | FileCheck %s --check-prefix CHECK11 +# CHECK11: Name: main.c diff --git a/lldb/tools/lldb-dap/JSONUtils.cpp b/lldb/tools/lldb-dap/JSONUtils.cpp index 3a63046d9a88..c8e5304ecec8 100644 --- a/lldb/tools/lldb-dap/JSONUtils.cpp +++ b/lldb/tools/lldb-dap/JSONUtils.cpp @@ -804,9 +804,11 @@ llvm::json::Value CreateStackFrame(lldb::SBFrame &frame) { llvm::raw_string_ostream os(frame_name); os << llvm::format_hex(frame.GetPC(), 18); } - bool is_optimized = frame.GetFunction().GetIsOptimized(); - if (is_optimized) + + // We only include `[opt]` if a custom frame format is not specified. + if (!g_dap.frame_format && frame.GetFunction().GetIsOptimized()) frame_name += " [opt]"; + EmplaceSafeString(object, "name", frame_name); auto source = CreateSource(frame); @@ -1101,6 +1103,29 @@ std::string CreateUniqueVariableNameForDisplay(lldb::SBValue v, // can use this optional information to present the // children in a paged UI and fetch them in chunks." // } +// "declaration": { +// "type": "object | undefined", +// "description": "Extension to the protocol that indicates the source +// location where the variable was declared. This value +// might not be present if no declaration is available.", +// "properties": { +// "path": { +// "type": "string | undefined", +// "description": "The source file path where the variable was +// declared." +// }, +// "line": { +// "type": "number | undefined", +// "description": "The 1-indexed source line where the variable was +// declared." +// }, +// "column": { +// "type": "number | undefined", +// "description": "The 1-indexed source column where the variable was +// declared." +// } +// } +// } // }, // "required": [ "name", "value", "variablesReference" ] // } @@ -1165,6 +1190,24 @@ llvm::json::Value CreateVariable(lldb::SBValue v, int64_t variablesReference, const char *evaluateName = evaluateStream.GetData(); if (evaluateName && evaluateName[0]) EmplaceSafeString(object, "evaluateName", std::string(evaluateName)); + + if (lldb::SBDeclaration decl = v.GetDeclaration(); decl.IsValid()) { + llvm::json::Object decl_obj; + if (lldb::SBFileSpec file = decl.GetFileSpec(); file.IsValid()) { + char path[PATH_MAX] = ""; + if (file.GetPath(path, sizeof(path)) && + lldb::SBFileSpec::ResolvePath(path, path, PATH_MAX)) { + decl_obj.try_emplace("path", std::string(path)); + } + } + + if (int line = decl.GetLine()) + decl_obj.try_emplace("line", line); + if (int column = decl.GetColumn()) + decl_obj.try_emplace("column", column); + + object.try_emplace("declaration", std::move(decl_obj)); + } return llvm::json::Value(std::move(object)); } diff --git a/llvm/CMakeLists.txt b/llvm/CMakeLists.txt index 1c983165b2ef..35be9bf16b5e 100644 --- a/llvm/CMakeLists.txt +++ b/llvm/CMakeLists.txt @@ -581,7 +581,9 @@ set(LLVM_TARGETS_TO_BUILD ${LLVM_EXPERIMENTAL_TARGETS_TO_BUILD}) list(REMOVE_DUPLICATES LLVM_TARGETS_TO_BUILD) -option(LLVM_ENABLE_PIC "Build Position-Independent Code" ON) +if (NOT CMAKE_SYSTEM_NAME MATCHES "OS390") + option(LLVM_ENABLE_PIC "Build Position-Independent Code" ON) +endif() option(LLVM_ENABLE_MODULES "Compile with C++ modules enabled." OFF) if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") option(LLVM_ENABLE_MODULE_DEBUGGING "Compile with -gmodules." ON) @@ -1127,6 +1129,8 @@ if (CMAKE_SYSTEM_NAME MATCHES "OS390") add_compile_definitions(_XOPEN_SOURCE=600) add_compile_definitions(_OPEN_SYS) # Needed for process information. add_compile_definitions(_OPEN_SYS_FILE_EXT) # Needed for EBCDIC I/O. + add_compile_definitions(_EXT) # Needed for file data. + add_compile_definitions(_UNIX03_THREADS) # Multithreading support. endif() # Build with _FILE_OFFSET_BITS=64 on Solaris to match g++ >= 9. diff --git a/llvm/cmake/modules/HandleLLVMOptions.cmake b/llvm/cmake/modules/HandleLLVMOptions.cmake index 97fd908d08f1..0699a8586fcc 100644 --- a/llvm/cmake/modules/HandleLLVMOptions.cmake +++ b/llvm/cmake/modules/HandleLLVMOptions.cmake @@ -988,6 +988,9 @@ if(LLVM_USE_SANITIZER) endif() # Prepare ASAN runtime if needed if (LLVM_USE_SANITIZER MATCHES ".*Address.*") + # lld string tail merging interacts badly with ASAN on Windows, turn it off here + # See https://github.com/llvm/llvm-project/issues/62078 + append("/opt:nolldtailmerge" CMAKE_EXE_LINKER_FLAGS CMAKE_MODULE_LINKER_FLAGS CMAKE_SHARED_LINKER_FLAGS) if (${CMAKE_MSVC_RUNTIME_LIBRARY} MATCHES "^(MultiThreaded|MultiThreadedDebug)$") append("/wholearchive:clang_rt.asan-${arch}.lib /wholearchive:clang_rt.asan_cxx-${arch}.lib" CMAKE_EXE_LINKER_FLAGS) diff --git a/llvm/docs/CoverageMappingFormat.rst b/llvm/docs/CoverageMappingFormat.rst index cbabad7a332c..c5cd1f795aab 100644 --- a/llvm/docs/CoverageMappingFormat.rst +++ b/llvm/docs/CoverageMappingFormat.rst @@ -178,7 +178,7 @@ defined inside macros, like this example demonstrates: Counter: ^^^^^^^^ -A coverage mapping counter can represents a reference to the profile +A coverage mapping counter can represent a reference to the profile instrumentation counter. The execution count for a region with such counter is determined by looking up the value of the corresponding profile instrumentation counter. diff --git a/llvm/docs/GettingStarted.rst b/llvm/docs/GettingStarted.rst index 24ed7118ee81..da9cc8aea6d3 100644 --- a/llvm/docs/GettingStarted.rst +++ b/llvm/docs/GettingStarted.rst @@ -43,7 +43,7 @@ Getting the Source Code and Building LLVM * You are likely only interested in the main branch moving forward, if you don't want `git fetch` (or `git pull`) to download user branches, use: - ``sed 's#fetch = +refs/heads/\*:refs/remotes/origin/\*#fetch = +refs/heads/main:refs/remotes/origin/main# -i llvm-project/.git/config`` + ``sed 's#fetch = +refs/heads/\*:refs/remotes/origin/\*#fetch = +refs/heads/main:refs/remotes/origin/main#' -i llvm-project/.git/config`` #. Configure and build LLVM and Clang: diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index f5e8065ca1dc..adda52b33c78 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -4299,6 +4299,11 @@ constants and smaller complex constants. "``< i32 42, i32 11, i32 74, i32 100 >``". Vector constants must have :ref:`vector type `, and the number and types of elements must match those specified by the type. + + When creating a vector whose elements have the same constant value, the + preferred syntax is ``splat ( Val)``. For example: "``splat (i32 11)``". + These vector constants must have ::ref:`vector type ` with an + element type that matches the ``splat`` operand. **Zero initialization** The string '``zeroinitializer``' can be used to zero initialize a value to zero of *any* type, including scalar and diff --git a/llvm/docs/MIRLangRef.rst b/llvm/docs/MIRLangRef.rst index 223701599ebb..52ff24daa7fb 100644 --- a/llvm/docs/MIRLangRef.rst +++ b/llvm/docs/MIRLangRef.rst @@ -58,12 +58,12 @@ for the post register allocation pseudo instruction expansion pass, you can specify the machine copy propagation pass in the ``-stop-after`` option, as it runs just before the pass that we are trying to test: - ``llc -stop-after=machine-cp bug-trigger.ll > test.mir`` + ``llc -stop-after=machine-cp bug-trigger.ll -o test.mir`` If the same pass is run multiple times, a run index can be included after the name with a comma. - ``llc -stop-after=dead-mi-elimination,1 bug-trigger.ll > test.mir`` + ``llc -stop-after=dead-mi-elimination,1 bug-trigger.ll -o test.mir`` After generating the input MIR file, you'll have to add a run line that uses the ``-run-pass`` option to it. In order to test the post register allocation diff --git a/llvm/docs/ReleaseNotes.rst b/llvm/docs/ReleaseNotes.rst index f58ae03a6efc..d5c634d2f29a 100644 --- a/llvm/docs/ReleaseNotes.rst +++ b/llvm/docs/ReleaseNotes.rst @@ -92,6 +92,15 @@ Changes to Interprocedural Optimizations Changes to the AArch64 Backend ------------------------------ +* Added support for Cortex-A520, Cortex-A720 and Cortex-X4 CPUs. + +* Neoverse-N2 was incorrectly marked as an Armv8.5a core. This has been + changed to an Armv9.0a core. However, crypto options are not enabled + by default for Armv9 cores, so `-mcpu=neoverse-n2+crypto` is now required + to enable crypto for this core. As far as the compiler is concerned, + Armv9.0a has the same features enabled as Armv8.5a, with the exception + of crypto. + Changes to the AMDGPU Backend ----------------------------- @@ -102,11 +111,11 @@ Changes to the AMDGPU Backend * Implemented :ref:`llvm.get.rounding ` -* Added support for Cortex-A520, Cortex-A720 and Cortex-X4 CPUs. - Changes to the ARM Backend -------------------------- +* Added support for Cortex-M52 CPUs. + Changes to the AVR Backend -------------------------- @@ -204,6 +213,19 @@ Changes to the C API on zext instructions, and ``LLVMGetIsDisjoint`` and ``LLVMSetIsDisjoint`` for getting/setting the new disjoint flag on or instructions. +* Added the following functions for manipulating operand bundles, as well as + building ``call`` and ``invoke`` instructions that use operand bundles: + + * ``LLVMBuildCallWithOperandBundles`` + * ``LLVMBuildInvokeWithOperandBundles`` + * ``LLVMCreateOperandBundle`` + * ``LLVMDisposeOperandBundle`` + * ``LLVMGetNumOperandBundles`` + * ``LLVMGetOperandBundleAtIndex`` + * ``LLVMGetNumOperandBundleArgs`` + * ``LLVMGetOperandBundleArgAtIndex`` + * ``LLVMGetOperandBundleTag`` + Changes to the CodeGen infrastructure ------------------------------------- diff --git a/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/LLJITWithRemoteDebugging.cpp b/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/LLJITWithRemoteDebugging.cpp index 291f14e1d7d7..900112506058 100644 --- a/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/LLJITWithRemoteDebugging.cpp +++ b/llvm/examples/OrcV2Examples/LLJITWithRemoteDebugging/LLJITWithRemoteDebugging.cpp @@ -88,6 +88,7 @@ #include "llvm/Support/InitLLVM.h" #include "llvm/Support/TargetSelect.h" #include "llvm/Support/raw_ostream.h" +#include "llvm/TargetParser/Host.h" #include "../ExampleModules.h" #include "RemoteJITUtils.h" @@ -173,24 +174,15 @@ int main(int argc, char *argv[]) { TSMs.push_back(ExitOnErr(parseExampleModuleFromFile(Path))); } - StringRef TT; + std::string TT; StringRef MainModuleName; TSMs.front().withModuleDo([&MainModuleName, &TT](Module &M) { MainModuleName = M.getName(); TT = M.getTargetTriple(); + if (TT.empty()) + TT = sys::getProcessTriple(); }); - for (const ThreadSafeModule &TSM : TSMs) - ExitOnErr(TSM.withModuleDo([TT, MainModuleName](Module &M) -> Error { - if (M.getTargetTriple() != TT) - return make_error( - formatv("Different target triples in input files:\n" - " '{0}' in '{1}'\n '{2}' in '{3}'", - TT, MainModuleName, M.getTargetTriple(), M.getName()), - inconvertibleErrorCode()); - return Error::success(); - })); - // Create a target machine that matches the input triple. JITTargetMachineBuilder JTMB((Triple(TT))); JTMB.setCodeModel(CodeModel::Small); diff --git a/llvm/include/llvm-c/Core.h b/llvm/include/llvm-c/Core.h index a575ec3709fe..7cb809d378c9 100644 --- a/llvm/include/llvm-c/Core.h +++ b/llvm/include/llvm-c/Core.h @@ -3000,6 +3000,74 @@ LLVMValueRef LLVMMDNodeInContext(LLVMContextRef C, LLVMValueRef *Vals, /** Deprecated: Use LLVMMDNodeInContext2 instead. */ LLVMValueRef LLVMMDNode(LLVMValueRef *Vals, unsigned Count); +/** + * @} + */ + +/** + * @defgroup LLVMCCoreOperandBundle Operand Bundles + * + * Functions in this group operate on LLVMOperandBundleRef instances that + * correspond to llvm::OperandBundleDef instances. + * + * @see llvm::OperandBundleDef + * + * @{ + */ + +/** + * Create a new operand bundle. + * + * Every invocation should be paired with LLVMDisposeOperandBundle() or memory + * will be leaked. + * + * @param Tag Tag name of the operand bundle + * @param TagLen Length of Tag + * @param Args Memory address of an array of bundle operands + * @param NumArgs Length of Args + */ +LLVMOperandBundleRef LLVMCreateOperandBundle(const char *Tag, size_t TagLen, + LLVMValueRef *Args, + unsigned NumArgs); + +/** + * Destroy an operand bundle. + * + * This must be called for every created operand bundle or memory will be + * leaked. + */ +void LLVMDisposeOperandBundle(LLVMOperandBundleRef Bundle); + +/** + * Obtain the tag of an operand bundle as a string. + * + * @param Bundle Operand bundle to obtain tag of. + * @param Len Out parameter which holds the length of the returned string. + * @return The tag name of Bundle. + * @see OperandBundleDef::getTag() + */ +const char *LLVMGetOperandBundleTag(LLVMOperandBundleRef Bundle, size_t *Len); + +/** + * Obtain the number of operands for an operand bundle. + * + * @param Bundle Operand bundle to obtain operand count of. + * @return The number of operands. + * @see OperandBundleDef::input_size() + */ +unsigned LLVMGetNumOperandBundleArgs(LLVMOperandBundleRef Bundle); + +/** + * Obtain the operand for an operand bundle at the given index. + * + * @param Bundle Operand bundle to obtain operand of. + * @param Index An operand index, must be less than + * LLVMGetNumOperandBundleArgs(). + * @return The operand. + */ +LLVMValueRef LLVMGetOperandBundleArgAtIndex(LLVMOperandBundleRef Bundle, + unsigned Index); + /** * @} */ @@ -3451,6 +3519,24 @@ LLVMTypeRef LLVMGetCalledFunctionType(LLVMValueRef C); */ LLVMValueRef LLVMGetCalledValue(LLVMValueRef Instr); +/** + * Obtain the number of operand bundles attached to this instruction. + * + * This only works on llvm::CallInst and llvm::InvokeInst instructions. + * + * @see llvm::CallBase::getNumOperandBundles() + */ +unsigned LLVMGetNumOperandBundles(LLVMValueRef C); + +/** + * Obtain the operand bundle attached to this instruction at the given index. + * Use LLVMDisposeOperandBundle to free the operand bundle. + * + * This only works on llvm::CallInst and llvm::InvokeInst instructions. + */ +LLVMOperandBundleRef LLVMGetOperandBundleAtIndex(LLVMValueRef C, + unsigned Index); + /** * Obtain whether a call instruction is a tail call. * @@ -3815,6 +3901,10 @@ LLVMValueRef LLVMBuildInvoke2(LLVMBuilderRef, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, const char *Name); +LLVMValueRef LLVMBuildInvokeWithOperandBundles( + LLVMBuilderRef, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, + unsigned NumArgs, LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, + LLVMOperandBundleRef *Bundles, unsigned NumBundles, const char *Name); LLVMValueRef LLVMBuildUnreachable(LLVMBuilderRef); /* Exception Handling */ @@ -4121,6 +4211,11 @@ LLVMValueRef LLVMBuildPhi(LLVMBuilderRef, LLVMTypeRef Ty, const char *Name); LLVMValueRef LLVMBuildCall2(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef Fn, LLVMValueRef *Args, unsigned NumArgs, const char *Name); +LLVMValueRef +LLVMBuildCallWithOperandBundles(LLVMBuilderRef, LLVMTypeRef, LLVMValueRef Fn, + LLVMValueRef *Args, unsigned NumArgs, + LLVMOperandBundleRef *Bundles, + unsigned NumBundles, const char *Name); LLVMValueRef LLVMBuildSelect(LLVMBuilderRef, LLVMValueRef If, LLVMValueRef Then, LLVMValueRef Else, const char *Name); diff --git a/llvm/include/llvm-c/LLJIT.h b/llvm/include/llvm-c/LLJIT.h index a06133aac4fb..a58c3b8bbef7 100644 --- a/llvm/include/llvm-c/LLJIT.h +++ b/llvm/include/llvm-c/LLJIT.h @@ -1,4 +1,4 @@ -/*===----------- llvm-c/LLJIT.h - OrcV2 LLJIT C bindings --------*- C++ -*-===*\ +/*===----------- llvm-c/LLJIT.h - OrcV2 LLJIT C bindings ----------*- C -*-===*\ |* *| |* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| |* Exceptions. *| diff --git a/llvm/include/llvm-c/LLJITUtils.h b/llvm/include/llvm-c/LLJITUtils.h new file mode 100644 index 000000000000..940097432b78 --- /dev/null +++ b/llvm/include/llvm-c/LLJITUtils.h @@ -0,0 +1,52 @@ +/*===------- llvm-c/LLJITUtils.h - Advanced LLJIT features --------*- C -*-===*\ +|* *| +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM *| +|* Exceptions. *| +|* See https://llvm.org/LICENSE.txt for license information. *| +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception *| +|* *| +|*===----------------------------------------------------------------------===*| +|* *| +|* This header declares the C interface for extra utilities to be used with *| +|* the LLJIT class from the llvm-c/LLJIT.h header. It requires to following *| +|* link libraries in addition to libLLVMOrcJIT.a: *| +|* - libLLVMOrcDebugging.a *| +|* *| +|* Many exotic languages can interoperate with C code but have a harder time *| +|* with C++ due to name mangling. So in addition to C, this interface enables *| +|* tools written in such languages. *| +|* *| +|* Note: This interface is experimental. It is *NOT* stable, and may be *| +|* changed without warning. Only C API usage documentation is *| +|* provided. See the C++ documentation for all higher level ORC API *| +|* details. *| +|* *| +\*===----------------------------------------------------------------------===*/ + +#ifndef LLVM_C_LLJITUTILS_H +#define LLVM_C_LLJITUTILS_H + +#include "llvm-c/LLJIT.h" + +LLVM_C_EXTERN_C_BEGIN + +/** + * @defgroup LLVMCExecutionEngineLLJITUtils LLJIT Utilities + * @ingroup LLVMCExecutionEngineLLJIT + * + * @{ + */ + +/** + * Install the plugin that submits debug objects to the executor. Executors must + * expose the llvm_orc_registerJITLoaderGDBWrapper symbol. + */ +LLVMErrorRef LLVMOrcLLJITEnableDebugSupport(LLVMOrcLLJITRef J); + +/** + * @} + */ + +LLVM_C_EXTERN_C_END + +#endif /* LLVM_C_LLJITUTILS_H */ diff --git a/llvm/include/llvm-c/Types.h b/llvm/include/llvm-c/Types.h index 4e9967372d79..d5474d986309 100644 --- a/llvm/include/llvm-c/Types.h +++ b/llvm/include/llvm-c/Types.h @@ -132,6 +132,11 @@ typedef struct LLVMOpaquePassManager *LLVMPassManagerRef; * @see llvm::Use */ typedef struct LLVMOpaqueUse *LLVMUseRef; +/** + * @see llvm::OperandBundleDef + */ +typedef struct LLVMOpaqueOperandBundle *LLVMOperandBundleRef; + /** * Used to represent an attributes. * diff --git a/llvm/include/llvm/ADT/SmallString.h b/llvm/include/llvm/ADT/SmallString.h index 0052c86fb37b..02fa28fc856d 100644 --- a/llvm/include/llvm/ADT/SmallString.h +++ b/llvm/include/llvm/ADT/SmallString.h @@ -120,15 +120,11 @@ public: /// @name String Predicates /// @{ - /// startswith - Check if this string starts with the given \p Prefix. - bool startswith(StringRef Prefix) const { - return str().startswith(Prefix); - } + /// starts_with - Check if this string starts with the given \p Prefix. + bool starts_with(StringRef Prefix) const { return str().starts_with(Prefix); } - /// endswith - Check if this string ends with the given \p Suffix. - bool endswith(StringRef Suffix) const { - return str().endswith(Suffix); - } + /// ends_with - Check if this string ends with the given \p Suffix. + bool ends_with(StringRef Suffix) const { return str().ends_with(Suffix); } /// @} /// @name String Searching diff --git a/llvm/include/llvm/ADT/SparseBitVector.h b/llvm/include/llvm/ADT/SparseBitVector.h index 1e00c1386187..7151af6146e6 100644 --- a/llvm/include/llvm/ADT/SparseBitVector.h +++ b/llvm/include/llvm/ADT/SparseBitVector.h @@ -15,8 +15,8 @@ #ifndef LLVM_ADT_SPARSEBITVECTOR_H #define LLVM_ADT_SPARSEBITVECTOR_H +#include "llvm/ADT/bit.h" #include "llvm/Support/ErrorHandling.h" -#include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" #include #include diff --git a/llvm/include/llvm/ADT/StringRef.h b/llvm/include/llvm/ADT/StringRef.h index 235a7b27c384..4e69d5b63354 100644 --- a/llvm/include/llvm/ADT/StringRef.h +++ b/llvm/include/llvm/ADT/StringRef.h @@ -264,12 +264,6 @@ namespace llvm { /// Check if this string starts with the given \p Prefix, ignoring case. [[nodiscard]] bool starts_with_insensitive(StringRef Prefix) const; - [[nodiscard]] LLVM_DEPRECATED( - "Use starts_with_insensitive instead", - "starts_with_insensitive") bool startswith_insensitive(StringRef Prefix) - const { - return starts_with_insensitive(Prefix); - } /// Check if this string ends with the given \p Suffix. [[nodiscard]] bool ends_with(StringRef Suffix) const { @@ -283,12 +277,6 @@ namespace llvm { /// Check if this string ends with the given \p Suffix, ignoring case. [[nodiscard]] bool ends_with_insensitive(StringRef Suffix) const; - [[nodiscard]] LLVM_DEPRECATED( - "Use ends_with_insensitive instead", - "ends_with_insensitive") bool endswith_insensitive(StringRef Suffix) - const { - return ends_with_insensitive(Suffix); - } /// @} /// @name String Searching diff --git a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h index 91848a91c17e..fd884f2a2f55 100644 --- a/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h +++ b/llvm/include/llvm/Analysis/ScalarEvolutionExpressions.h @@ -16,7 +16,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/iterator_range.h" #include "llvm/Analysis/ScalarEvolution.h" #include "llvm/IR/Constants.h" #include "llvm/IR/ValueHandle.h" diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h index 8635bdd470ee..fb6f3287e3d2 100644 --- a/llvm/include/llvm/Analysis/TargetTransformInfo.h +++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h @@ -2376,12 +2376,12 @@ public: bool IsZeroCmp) const override { return Impl.enableMemCmpExpansion(OptSize, IsZeroCmp); } - bool enableInterleavedAccessVectorization() override { - return Impl.enableInterleavedAccessVectorization(); - } bool enableSelectOptimize() override { return Impl.enableSelectOptimize(); } + bool enableInterleavedAccessVectorization() override { + return Impl.enableInterleavedAccessVectorization(); + } bool enableMaskedInterleavedAccessVectorization() override { return Impl.enableMaskedInterleavedAccessVectorization(); } diff --git a/llvm/include/llvm/AsmParser/LLParser.h b/llvm/include/llvm/AsmParser/LLParser.h index 793287c772b5..54bc3e582e01 100644 --- a/llvm/include/llvm/AsmParser/LLParser.h +++ b/llvm/include/llvm/AsmParser/LLParser.h @@ -51,15 +51,23 @@ namespace llvm { /// or a symbolic (%var) reference. This is just a discriminated union. struct ValID { enum { - t_LocalID, t_GlobalID, // ID in UIntVal. - t_LocalName, t_GlobalName, // Name in StrVal. - t_APSInt, t_APFloat, // Value in APSIntVal/APFloatVal. - t_Null, t_Undef, t_Zero, t_None, t_Poison, // No value. - t_EmptyArray, // No value: [] - t_Constant, // Value in ConstantVal. - t_InlineAsm, // Value in FTy/StrVal/StrVal2/UIntVal. - t_ConstantStruct, // Value in ConstantStructElts. - t_PackedConstantStruct // Value in ConstantStructElts. + t_LocalID, // ID in UIntVal. + t_GlobalID, // ID in UIntVal. + t_LocalName, // Name in StrVal. + t_GlobalName, // Name in StrVal. + t_APSInt, // Value in APSIntVal. + t_APFloat, // Value in APFloatVal. + t_Null, // No value. + t_Undef, // No value. + t_Zero, // No value. + t_None, // No value. + t_Poison, // No value. + t_EmptyArray, // No value: [] + t_Constant, // Value in ConstantVal. + t_ConstantSplat, // Value in ConstantVal. + t_InlineAsm, // Value in FTy/StrVal/StrVal2/UIntVal. + t_ConstantStruct, // Value in ConstantStructElts. + t_PackedConstantStruct // Value in ConstantStructElts. } Kind = t_LocalID; LLLexer::LocTy Loc; @@ -407,9 +415,10 @@ namespace llvm { std::map, WholeProgramDevirtResolution::ByArg> &ResByArg); bool parseArgs(std::vector &Args); - void addGlobalValueToIndex(std::string Name, GlobalValue::GUID, + bool addGlobalValueToIndex(std::string Name, GlobalValue::GUID, GlobalValue::LinkageTypes Linkage, unsigned ID, - std::unique_ptr Summary); + std::unique_ptr Summary, + LocTy Loc); bool parseOptionalAllocs(std::vector &Allocs); bool parseMemProfs(std::vector &MIBs); bool parseAllocType(uint8_t &AllocType); diff --git a/llvm/include/llvm/AsmParser/LLToken.h b/llvm/include/llvm/AsmParser/LLToken.h index 0aa0093e8efb..147cf56c821a 100644 --- a/llvm/include/llvm/AsmParser/LLToken.h +++ b/llvm/include/llvm/AsmParser/LLToken.h @@ -336,6 +336,7 @@ enum Kind { kw_extractelement, kw_insertelement, kw_shufflevector, + kw_splat, kw_extractvalue, kw_insertvalue, kw_blockaddress, diff --git a/llvm/include/llvm/BinaryFormat/DynamicTags.def b/llvm/include/llvm/BinaryFormat/DynamicTags.def index f393b82406b4..1502d375f5c4 100644 --- a/llvm/include/llvm/BinaryFormat/DynamicTags.def +++ b/llvm/include/llvm/BinaryFormat/DynamicTags.def @@ -132,6 +132,12 @@ AARCH64_DYNAMIC_TAG(AARCH64_MEMTAG_STACK, 0x7000000c) AARCH64_DYNAMIC_TAG(AARCH64_MEMTAG_GLOBALS, 0x7000000d) AARCH64_DYNAMIC_TAG(AARCH64_MEMTAG_GLOBALSSZ, 0x7000000f) +// AArch64 specific dynamic table entries for RELR auth relocations as described here: +// https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#dynamic-section +AARCH64_DYNAMIC_TAG(AARCH64_AUTH_RELRSZ, 0x70000011) +AARCH64_DYNAMIC_TAG(AARCH64_AUTH_RELR, 0x70000012) +AARCH64_DYNAMIC_TAG(AARCH64_AUTH_RELRENT, 0x70000013) + // Hexagon specific dynamic table entries HEXAGON_DYNAMIC_TAG(HEXAGON_SYMSZ, 0x70000000) HEXAGON_DYNAMIC_TAG(HEXAGON_VER, 0x70000001) diff --git a/llvm/include/llvm/BinaryFormat/ELF.h b/llvm/include/llvm/BinaryFormat/ELF.h index d1ce8e20b4be..40c795410f95 100644 --- a/llvm/include/llvm/BinaryFormat/ELF.h +++ b/llvm/include/llvm/BinaryFormat/ELF.h @@ -1060,6 +1060,9 @@ enum : unsigned { SHT_ARM_ATTRIBUTES = 0x70000003U, SHT_ARM_DEBUGOVERLAY = 0x70000004U, SHT_ARM_OVERLAYSECTION = 0x70000005U, + // Special aarch64-specific section for MTE support, as described in: + // https://github.com/ARM-software/abi-aa/blob/main/pauthabielf64/pauthabielf64.rst#section-types + SHT_AARCH64_AUTH_RELR = 0x70000004U, // Special aarch64-specific sections for MTE support, as described in: // https://github.com/ARM-software/abi-aa/blob/main/memtagabielf64/memtagabielf64.rst#7section-types SHT_AARCH64_MEMTAG_GLOBALS_STATIC = 0x70000007U, @@ -1647,6 +1650,11 @@ enum { NT_ANDROID_TYPE_MEMTAG = 4, }; +// ARM note types. +enum { + NT_ARM_TYPE_PAUTH_ABI_TAG = 1, +}; + // Memory tagging values used in NT_ANDROID_TYPE_MEMTAG notes. enum { // Enumeration to determine the tagging mode. In Android-land, 'SYNC' means diff --git a/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def b/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def index b507109b19e1..30375de420e3 100644 --- a/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def +++ b/llvm/include/llvm/BinaryFormat/ELFRelocs/AArch64.def @@ -121,6 +121,7 @@ ELF_RELOC(R_AARCH64_TLSLE_LDST128_TPREL_LO12, 0x23a) ELF_RELOC(R_AARCH64_TLSLE_LDST128_TPREL_LO12_NC, 0x23b) ELF_RELOC(R_AARCH64_TLSLD_LDST128_DTPREL_LO12, 0x23c) ELF_RELOC(R_AARCH64_TLSLD_LDST128_DTPREL_LO12_NC, 0x23d) +ELF_RELOC(R_AARCH64_AUTH_ABS64, 0x244) // Dynamic relocations start ELF_RELOC(R_AARCH64_COPY, 0x400) ELF_RELOC(R_AARCH64_GLOB_DAT, 0x401) @@ -134,7 +135,7 @@ ELF_RELOC(R_AARCH64_TLS_DTPREL64, 0x405) ELF_RELOC(R_AARCH64_TLS_TPREL64, 0x406) ELF_RELOC(R_AARCH64_TLSDESC, 0x407) ELF_RELOC(R_AARCH64_IRELATIVE, 0x408) -ELF_RELOC(R_AARCH64_AUTH_ABS64, 0xe100) +ELF_RELOC(R_AARCH64_AUTH_RELATIVE, 0x411) // ELF_RELOC(R_AARCH64_P32_NONE, 0) ELF_RELOC(R_AARCH64_P32_ABS32, 0x001) diff --git a/llvm/include/llvm/CodeGen/CallingConvLower.h b/llvm/include/llvm/CodeGen/CallingConvLower.h index 0989fae54b3a..932a2a94ab1f 100644 --- a/llvm/include/llvm/CodeGen/CallingConvLower.h +++ b/llvm/include/llvm/CodeGen/CallingConvLower.h @@ -14,6 +14,7 @@ #ifndef LLVM_CODEGEN_CALLINGCONVLOWER_H #define LLVM_CODEGEN_CALLINGCONVLOWER_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/Register.h" #include "llvm/CodeGen/TargetCallingConv.h" diff --git a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h index 076719abd035..bb139ef2eb35 100644 --- a/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h +++ b/llvm/include/llvm/CodeGen/CodeGenPassBuilder.h @@ -23,7 +23,9 @@ #include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/Analysis/TypeBasedAliasAnalysis.h" #include "llvm/CodeGen/CallBrPrepare.h" +#include "llvm/CodeGen/DwarfEHPrepare.h" #include "llvm/CodeGen/ExpandReductions.h" +#include "llvm/CodeGen/InterleavedAccess.h" #include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/PreISelIntrinsicLowering.h" #include "llvm/CodeGen/ReplaceWithVeclib.h" @@ -677,14 +679,14 @@ void CodeGenPassBuilder::addPassesToHandleExceptions( case ExceptionHandling::DwarfCFI: case ExceptionHandling::ARM: case ExceptionHandling::AIX: - addPass(DwarfEHPass(getOptLevel())); + addPass(DwarfEHPreparePass(&TM)); break; case ExceptionHandling::WinEH: // We support using both GCC-style and MSVC-style exceptions on Windows, so // add both preparation passes. Each pass will only actually run if it // recognizes the personality function. addPass(WinEHPreparePass()); - addPass(DwarfEHPass(getOptLevel())); + addPass(DwarfEHPreparePass(&TM)); break; case ExceptionHandling::Wasm: // Wasm EH uses Windows EH instructions, but it does not need to demote PHIs diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h index 0b167ce9650d..e423e48fd31e 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegacyLegalizerInfo.h @@ -19,6 +19,7 @@ #include "llvm/CodeGen/LowLevelType.h" #include "llvm/CodeGen/TargetOpcodes.h" #include +#include namespace llvm { struct LegalityQuery; diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h index 365d2223a81c..711ba10247c3 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h @@ -432,6 +432,7 @@ public: LegalizeResult lowerVectorReduction(MachineInstr &MI); LegalizeResult lowerMemcpyInline(MachineInstr &MI); LegalizeResult lowerMemCpyFamily(MachineInstr &MI, unsigned MaxLen = 0); + LegalizeResult lowerVAArg(MachineInstr &MI); }; /// Helper function that creates a libcall to the given \p Name using the given diff --git a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h index e0101a5ac1ca..1387a0a37561 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/MachineIRBuilder.h @@ -1194,7 +1194,7 @@ public: const SrcOp &Op0, const SrcOp &Op1, std::optional Flags = std::nullopt); - /// Build and insert a \p Res = G_IS_FPCLASS \p Pred, \p Src, \p Mask + /// Build and insert a \p Res = G_IS_FPCLASS \p Src, \p Mask MachineInstrBuilder buildIsFPClass(const DstOp &Res, const SrcOp &Src, unsigned Mask) { return buildInstr(TargetOpcode::G_IS_FPCLASS, {Res}, @@ -1529,6 +1529,11 @@ public: /// Build and insert `G_FENCE Ordering, Scope`. MachineInstrBuilder buildFence(unsigned Ordering, unsigned Scope); + /// Build and insert G_PREFETCH \p Addr, \p RW, \p Locality, \p CacheType + MachineInstrBuilder buildPrefetch(const SrcOp &Addr, unsigned RW, + unsigned Locality, unsigned CacheType, + MachineMemOperand &MMO); + /// Build and insert \p Dst = G_FREEZE \p Src MachineInstrBuilder buildFreeze(const DstOp &Dst, const SrcOp &Src) { return buildInstr(TargetOpcode::G_FREEZE, {Dst}, {Src}); diff --git a/llvm/include/llvm/CodeGen/InterleavedAccess.h b/llvm/include/llvm/CodeGen/InterleavedAccess.h new file mode 100644 index 000000000000..31bd19a3191a --- /dev/null +++ b/llvm/include/llvm/CodeGen/InterleavedAccess.h @@ -0,0 +1,34 @@ +//===---- llvm/CodeGen/InterleavedAccess.h ----------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file contains the declaration of the InterleavedAccessPass class, +/// its corresponding pass name is `interleaved-access`. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_CODEGEN_INTERLEAVEDACCESS_H +#define LLVM_CODEGEN_INTERLEAVEDACCESS_H + +#include "llvm/IR/PassManager.h" + +namespace llvm { + +class TargetMachine; + +class InterleavedAccessPass : public PassInfoMixin { + const TargetMachine *TM; + +public: + explicit InterleavedAccessPass(const TargetMachine *TM) : TM(TM) {} + PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM); +}; + +} // namespace llvm + +#endif // LLVM_CODEGEN_INTERLEAVEDACCESS_H diff --git a/llvm/include/llvm/CodeGen/MachinePassRegistry.def b/llvm/include/llvm/CodeGen/MachinePassRegistry.def index 1e9e5838841b..e6e979a4582c 100644 --- a/llvm/include/llvm/CodeGen/MachinePassRegistry.def +++ b/llvm/include/llvm/CodeGen/MachinePassRegistry.def @@ -38,11 +38,13 @@ FUNCTION_ANALYSIS("targetir", TargetIRAnalysis, #endif FUNCTION_PASS("callbrprepare", CallBrPreparePass, ()) FUNCTION_PASS("consthoist", ConstantHoistingPass, ()) +FUNCTION_PASS("dwarf-eh-prepare", DwarfEHPreparePass, (TM)) FUNCTION_PASS("ee-instrument", EntryExitInstrumenterPass, (false)) FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass, ()) FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass, ()) FUNCTION_PASS("expand-reductions", ExpandReductionsPass, ()) FUNCTION_PASS("expandvp", ExpandVectorPredicationPass, ()) +FUNCTION_PASS("interleaved-access", InterleavedAccessPass, (TM)) FUNCTION_PASS("lower-constant-intrinsics", LowerConstantIntrinsicsPass, ()) FUNCTION_PASS("lowerinvoke", LowerInvokePass, ()) FUNCTION_PASS("mergeicmps", MergeICmpsPass, ()) @@ -122,12 +124,10 @@ DUMMY_FUNCTION_PASS("atomic-expand", AtomicExpandPass, ()) DUMMY_FUNCTION_PASS("cfguard-check", CFGuardCheckPass, ()) DUMMY_FUNCTION_PASS("cfguard-dispatch", CFGuardDispatchPass, ()) DUMMY_FUNCTION_PASS("codegenprepare", CodeGenPreparePass, ()) -DUMMY_FUNCTION_PASS("dwarfehprepare", DwarfEHPass, ()) DUMMY_FUNCTION_PASS("expandmemcmp", ExpandMemCmpPass, ()) DUMMY_FUNCTION_PASS("gc-info-printer", GCInfoPrinterPass, ()) DUMMY_FUNCTION_PASS("gc-lowering", GCLoweringPass, ()) DUMMY_FUNCTION_PASS("indirectbr-expand", IndirectBrExpandPass, ()) -DUMMY_FUNCTION_PASS("interleaved-access", InterleavedAccessPass, ()) DUMMY_FUNCTION_PASS("select-optimize", SelectOptimizePass, ()) DUMMY_FUNCTION_PASS("shadow-stack-gc-lowering", ShadowStackGCLoweringPass, ()) DUMMY_FUNCTION_PASS("sjljehprepare", SjLjEHPreparePass, ()) diff --git a/llvm/include/llvm/CodeGen/SchedulerRegistry.h b/llvm/include/llvm/CodeGen/SchedulerRegistry.h index 0c356e62ae4e..cf648d1316c9 100644 --- a/llvm/include/llvm/CodeGen/SchedulerRegistry.h +++ b/llvm/include/llvm/CodeGen/SchedulerRegistry.h @@ -63,7 +63,7 @@ public: ScheduleDAGSDNodes *createBURRListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel); -/// createBURRListDAGScheduler - This creates a bottom up list scheduler that +/// createSourceListDAGScheduler - This creates a bottom up list scheduler that /// schedules nodes in source code order when possible. ScheduleDAGSDNodes *createSourceListDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel OptLevel); diff --git a/llvm/include/llvm/CodeGen/TargetSchedule.h b/llvm/include/llvm/CodeGen/TargetSchedule.h index 3d39798790cd..bfe4234abf8e 100644 --- a/llvm/include/llvm/CodeGen/TargetSchedule.h +++ b/llvm/include/llvm/CodeGen/TargetSchedule.h @@ -46,7 +46,7 @@ class TargetSchedModel { unsigned computeInstrLatency(const MCSchedClassDesc &SCDesc) const; public: - TargetSchedModel() : SchedModel(MCSchedModel::GetDefaultSchedModel()) {} + TargetSchedModel() : SchedModel(MCSchedModel::Default) {} /// Initialize the machine model for instruction scheduling. /// diff --git a/llvm/include/llvm/DebugInfo/CodeView/CodeView.h b/llvm/include/llvm/DebugInfo/CodeView/CodeView.h index 62e559e2ceba..0bfd06e05bd8 100644 --- a/llvm/include/llvm/DebugInfo/CodeView/CodeView.h +++ b/llvm/include/llvm/DebugInfo/CodeView/CodeView.h @@ -16,6 +16,7 @@ #include #include +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/Support/Endian.h" namespace llvm { @@ -51,15 +52,15 @@ enum SymbolKind : uint16_t { #define CV_DEFINE_ENUM_CLASS_FLAGS_OPERATORS(Class) \ inline Class operator|(Class a, Class b) { \ - return static_cast(static_cast>(a) | \ - static_cast>(b)); \ + return static_cast(llvm::to_underlying(a) | \ + llvm::to_underlying(b)); \ } \ inline Class operator&(Class a, Class b) { \ - return static_cast(static_cast>(a) & \ - static_cast>(b)); \ + return static_cast(llvm::to_underlying(a) & \ + llvm::to_underlying(b)); \ } \ inline Class operator~(Class a) { \ - return static_cast(~static_cast>(a)); \ + return static_cast(~llvm::to_underlying(a)); \ } \ inline Class &operator|=(Class &a, Class b) { \ a = a | b; \ diff --git a/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h b/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h index 01de8b49dd78..0adbd25cb369 100644 --- a/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h +++ b/llvm/include/llvm/DebugInfo/PDB/Native/FormatUtil.h @@ -10,6 +10,7 @@ #define LLVM_DEBUGINFO_PDB_NATIVE_FORMATUTIL_H #include "llvm/ADT/ArrayRef.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/ADT/StringRef.h" #include "llvm/DebugInfo/CodeView/CodeView.h" #include "llvm/Support/Endian.h" @@ -34,8 +35,7 @@ namespace pdb { return Ret; template std::string formatUnknownEnum(T Value) { - return formatv("unknown ({0})", static_cast>(Value)) - .str(); + return formatv("unknown ({0})", llvm::to_underlying(Value)).str(); } std::string formatSegmentOffset(uint16_t Segment, uint32_t Offset); diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h index 27a90ebef3d6..40b9339eb531 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch64.h @@ -646,7 +646,7 @@ inline Symbol &createAnonymousPointer(LinkGraph &G, Section &PointerSection, inline Block &createPointerJumpStubBlock(LinkGraph &G, Section &StubSection, Symbol &PointerSymbol) { auto &B = G.createContentBlock(StubSection, PointerJumpStubContent, - orc::ExecutorAddr(~uint64_t(11)), 1, 0); + orc::ExecutorAddr(~uint64_t(11)), 4, 0); B.addEdge(Page21, 0, PointerSymbol, 0); B.addEdge(PageOffset12, 4, PointerSymbol, 0); return B; diff --git a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h index 7203b80052b5..db7855e69a66 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/MachOPlatform.h @@ -47,6 +47,15 @@ public: LLVM_MARK_AS_BITMASK_ENUM(/* LargestValue = */ Callable) }; + /// Used by setupJITDylib to create MachO header MaterializationUnits for + /// JITDylibs. + using MachOHeaderMUBuilder = + unique_function(MachOPlatform &MOP)>; + + /// Simple MachO header graph builder. + static inline std::unique_ptr + buildSimpleMachOHeaderMU(MachOPlatform &MOP); + /// Try to create a MachOPlatform instance, adding the ORC runtime to the /// given JITDylib. /// @@ -88,17 +97,23 @@ public: static Expected> Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, std::unique_ptr OrcRuntime, + MachOHeaderMUBuilder BuildMachOHeaderMU = buildSimpleMachOHeaderMU, std::optional RuntimeAliases = std::nullopt); /// Construct using a path to the ORC runtime. static Expected> Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, const char *OrcRuntimePath, + MachOHeaderMUBuilder BuildMachOHeaderMU = buildSimpleMachOHeaderMU, std::optional RuntimeAliases = std::nullopt); ExecutionSession &getExecutionSession() const { return ES; } ObjectLinkingLayer &getObjectLinkingLayer() const { return ObjLinkingLayer; } + NonOwningSymbolStringPtr getMachOHeaderStartSymbol() const { + return NonOwningSymbolStringPtr(MachOHeaderStartSymbol); + } + Error setupJITDylib(JITDylib &JD) override; Error teardownJITDylib(JITDylib &JD) override; Error notifyAdding(ResourceTracker &RT, @@ -118,6 +133,9 @@ public: standardRuntimeUtilityAliases(); private: + using SymbolTableVector = SmallVector< + std::tuple>; + // Data needed for bootstrap only. struct BootstrapInfo { std::mutex Mutex; @@ -125,6 +143,7 @@ private: size_t ActiveGraphs = 0; shared::AllocActions DeferredAAs; ExecutorAddr MachOHeaderAddr; + SymbolTableVector SymTab; }; // The MachOPlatformPlugin scans/modifies LinkGraphs to support MachO @@ -239,7 +258,7 @@ private: MachOPlatform(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, std::unique_ptr OrcRuntimeGenerator, - Error &Err); + MachOHeaderMUBuilder BuildMachOHeaderMU, Error &Err); // Associate MachOPlatform JIT-side runtime support functions with handlers. Error associateRuntimeSupportFunctions(); @@ -267,6 +286,7 @@ private: ExecutionSession &ES; JITDylib &PlatformJD; ObjectLinkingLayer &ObjLinkingLayer; + MachOHeaderMUBuilder BuildMachOHeaderMU; SymbolStringPtr MachOHeaderStartSymbol = ES.intern("___dso_handle"); @@ -313,6 +333,49 @@ private: std::atomic Bootstrap; }; +// Generates a MachO header. +class SimpleMachOHeaderMU : public MaterializationUnit { +public: + SimpleMachOHeaderMU(MachOPlatform &MOP, SymbolStringPtr HeaderStartSymbol); + StringRef getName() const override { return "MachOHeaderMU"; } + void materialize(std::unique_ptr R) override; + void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override; + +protected: + virtual jitlink::Block &createHeaderBlock(JITDylib &JD, jitlink::LinkGraph &G, + jitlink::Section &HeaderSection); + +private: + struct HeaderSymbol { + const char *Name; + uint64_t Offset; + }; + + static constexpr HeaderSymbol AdditionalHeaderSymbols[] = { + {"___mh_executable_header", 0}}; + + void addMachOHeader(JITDylib &JD, jitlink::LinkGraph &G, + const SymbolStringPtr &InitializerSymbol); + static MaterializationUnit::Interface + createHeaderInterface(MachOPlatform &MOP, + const SymbolStringPtr &HeaderStartSymbol); + + MachOPlatform &MOP; +}; + +/// Simple MachO header graph builder. +inline std::unique_ptr +MachOPlatform::buildSimpleMachOHeaderMU(MachOPlatform &MOP) { + return std::make_unique(MOP, MOP.MachOHeaderStartSymbol); +} + +struct MachOHeaderInfo { + size_t PageSize = 0; + uint32_t CPUType = 0; + uint32_t CPUSubType = 0; +}; +MachOHeaderInfo getMachOHeaderInfoFromTriple(const Triple &TT); + } // end namespace orc } // end namespace llvm diff --git a/llvm/include/llvm/Frontend/HLSL/HLSLResource.h b/llvm/include/llvm/Frontend/HLSL/HLSLResource.h index ba08ee519351..eedecaea4e58 100644 --- a/llvm/include/llvm/Frontend/HLSL/HLSLResource.h +++ b/llvm/include/llvm/Frontend/HLSL/HLSLResource.h @@ -59,15 +59,16 @@ class FrontendResource { public: FrontendResource(MDNode *E) : Entry(E) { - assert(Entry->getNumOperands() == 5 && "Unexpected metadata shape"); + assert(Entry->getNumOperands() == 6 && "Unexpected metadata shape"); } FrontendResource(GlobalVariable *GV, StringRef TypeStr, ResourceKind RK, - uint32_t ResIndex, uint32_t Space); + bool IsROV, uint32_t ResIndex, uint32_t Space); GlobalVariable *getGlobalVariable(); StringRef getSourceType(); - uint32_t getResourceKind(); + ResourceKind getResourceKind(); + bool getIsROV(); uint32_t getResourceIndex(); uint32_t getSpace(); MDNode *getMetadata() { return Entry; } diff --git a/llvm/include/llvm/Frontend/Offloading/Utility.h b/llvm/include/llvm/Frontend/Offloading/Utility.h index 631d5a5a3db6..520c192996a0 100644 --- a/llvm/include/llvm/Frontend/Offloading/Utility.h +++ b/llvm/include/llvm/Frontend/Offloading/Utility.h @@ -6,12 +6,35 @@ // //===----------------------------------------------------------------------===// +#ifndef LLVM_FRONTEND_OFFLOADING_UTILITY_H +#define LLVM_FRONTEND_OFFLOADING_UTILITY_H + #include "llvm/IR/Module.h" #include "llvm/Object/OffloadBinary.h" namespace llvm { namespace offloading { +/// Offloading entry flags for CUDA / HIP. The first three bits indicate the +/// type of entry while the others are a bit field for additional information. +enum OffloadEntryKindFlag : uint32_t { + /// Mark the entry as a global entry. This indicates the presense of a + /// kernel if the size size field is zero and a variable otherwise. + OffloadGlobalEntry = 0x0, + /// Mark the entry as a managed global variable. + OffloadGlobalManagedEntry = 0x1, + /// Mark the entry as a surface variable. + OffloadGlobalSurfaceEntry = 0x2, + /// Mark the entry as a texture variable. + OffloadGlobalTextureEntry = 0x3, + /// Mark the entry as being extern. + OffloadGlobalExtern = 0x1 << 3, + /// Mark the entry as being constant. + OffloadGlobalConstant = 0x1 << 4, + /// Mark the entry as being a normalized surface. + OffloadGlobalNormalized = 0x1 << 5, +}; + /// Returns the type of the offloading entry we use to store kernels and /// globals that will be registered with the offloading runtime. StructType *getEntryTy(Module &M); @@ -25,7 +48,7 @@ StructType *getEntryTy(Module &M); /// char *name; // Name of the function or global. /// size_t size; // Size of the entry info (0 if it a function). /// int32_t flags; -/// int32_t reserved; +/// int32_t data; /// }; /// /// \param M The module to be used @@ -33,9 +56,11 @@ StructType *getEntryTy(Module &M); /// \param Name The symbol name associated with the global. /// \param Size The size in bytes of the global (0 for functions). /// \param Flags Flags associated with the entry. +/// \param Data Extra data storage associated with the entry. /// \param SectionName The section this entry will be placed at. void emitOffloadingEntry(Module &M, Constant *Addr, StringRef Name, - uint64_t Size, int32_t Flags, StringRef SectionName); + uint64_t Size, int32_t Flags, int32_t Data, + StringRef SectionName); /// Creates a pair of globals used to iterate the array of offloading entries by /// accessing the section variables provided by the linker. @@ -44,3 +69,5 @@ getOffloadEntryArray(Module &M, StringRef SectionName); } // namespace offloading } // namespace llvm + +#endif // LLVM_FRONTEND_OFFLOADING_UTILITY_H diff --git a/llvm/include/llvm/IR/DebugProgramInstruction.h b/llvm/include/llvm/IR/DebugProgramInstruction.h index d6b4536a2a07..f73a6237a477 100644 --- a/llvm/include/llvm/IR/DebugProgramInstruction.h +++ b/llvm/include/llvm/IR/DebugProgramInstruction.h @@ -113,7 +113,7 @@ public: /// Directly construct a new DPValue representing a dbg.value intrinsic /// assigning \p Location to the DV / Expr / DI variable. DPValue(Metadata *Location, DILocalVariable *DV, DIExpression *Expr, - const DILocation *DI); + const DILocation *DI, LocationType Type = LocationType::Value); /// Iterator for ValueAsMetadata that internally uses direct pointer iteration /// over either a ValueAsMetadata* or a ValueAsMetadata**, dereferencing to the diff --git a/llvm/include/llvm/IR/Dominators.h b/llvm/include/llvm/IR/Dominators.h index b6b737f34b5e..8784a425d284 100644 --- a/llvm/include/llvm/IR/Dominators.h +++ b/llvm/include/llvm/IR/Dominators.h @@ -16,7 +16,6 @@ #include "llvm/ADT/APInt.h" #include "llvm/ADT/ArrayRef.h" -#include "llvm/ADT/DenseMap.h" #include "llvm/ADT/DenseMapInfo.h" #include "llvm/ADT/DepthFirstIterator.h" #include "llvm/ADT/Hashing.h" @@ -24,7 +23,6 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Twine.h" #include "llvm/ADT/ilist_iterator.h" -#include "llvm/ADT/iterator_range.h" #include "llvm/IR/BasicBlock.h" #include "llvm/IR/CFG.h" #include "llvm/IR/PassManager.h" diff --git a/llvm/include/llvm/IR/GetElementPtrTypeIterator.h b/llvm/include/llvm/IR/GetElementPtrTypeIterator.h index 8c6ede96c873..f3272327c3f8 100644 --- a/llvm/include/llvm/IR/GetElementPtrTypeIterator.h +++ b/llvm/include/llvm/IR/GetElementPtrTypeIterator.h @@ -20,7 +20,6 @@ #include "llvm/IR/Operator.h" #include "llvm/IR/User.h" #include "llvm/Support/Casting.h" -#include #include #include #include diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 06f9c0445bce..bc9f99783d98 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -2105,7 +2105,8 @@ def int_amdgcn_wqm_vote : Intrinsic<[llvm_i1_ty], def int_amdgcn_kill : Intrinsic<[], [llvm_i1_ty], [IntrNoCallback, IntrNoFree]>; def int_amdgcn_endpgm : ClangBuiltin<"__builtin_amdgcn_endpgm">, - Intrinsic<[], [], [IntrNoReturn, IntrCold, IntrNoMem, IntrHasSideEffects, IntrNoCallback, IntrNoFree] + Intrinsic<[], [], [IntrNoReturn, IntrCold, IntrNoMem, IntrHasSideEffects, IntrConvergent, + IntrNoCallback, IntrNoFree] >; // If false, mark all active lanes as helper lanes until the end of program. diff --git a/llvm/include/llvm/IR/OperandTraits.h b/llvm/include/llvm/IR/OperandTraits.h index 979ad35019f8..ffece6324aab 100644 --- a/llvm/include/llvm/IR/OperandTraits.h +++ b/llvm/include/llvm/IR/OperandTraits.h @@ -94,10 +94,10 @@ struct VariadicOperandTraits { template struct HungoffOperandTraits { static Use *op_begin(User* U) { - return U->getOperandList(); + return U->getHungOffOperands(); } static Use *op_end(User* U) { - return U->getOperandList() + U->getNumOperands(); + return U->getHungOffOperands() + U->getNumOperands(); } static unsigned operands(const User *U) { return U->getNumOperands(); diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h index 07f950a9f452..096d1688af3f 100644 --- a/llvm/include/llvm/IR/PatternMatch.h +++ b/llvm/include/llvm/IR/PatternMatch.h @@ -1270,6 +1270,14 @@ inline DisjointOr_match m_c_DisjointOr(const LHS &L, return DisjointOr_match(L, R); } +/// Match either "and" or "or disjoint". +template +inline match_combine_or, + DisjointOr_match> +m_AddLike(const LHS &L, const RHS &R) { + return m_CombineOr(m_Add(L, R), m_DisjointOr(L, R)); +} + //===----------------------------------------------------------------------===// // Class that matches a group of binary opcodes. // diff --git a/llvm/include/llvm/MC/MCAsmBackend.h b/llvm/include/llvm/MC/MCAsmBackend.h index ebb33d0ab61f..8931e8cab2fa 100644 --- a/llvm/include/llvm/MC/MCAsmBackend.h +++ b/llvm/include/llvm/MC/MCAsmBackend.h @@ -101,7 +101,8 @@ public: /// Hook to check if a relocation is needed for some target specific reason. virtual bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { return false; } @@ -124,7 +125,8 @@ public: virtual bool evaluateTargetFixup(const MCAssembler &Asm, const MCAsmLayout &Layout, const MCFixup &Fixup, const MCFragment *DF, - const MCValue &Target, uint64_t &Value, + const MCValue &Target, + const MCSubtargetInfo *STI, uint64_t &Value, bool &WasForced) { llvm_unreachable("Need to implement hook if target has custom fixups"); } diff --git a/llvm/include/llvm/MC/MCAssembler.h b/llvm/include/llvm/MC/MCAssembler.h index 5e1fc738b1da..5ae5f6d70938 100644 --- a/llvm/include/llvm/MC/MCAssembler.h +++ b/llvm/include/llvm/MC/MCAssembler.h @@ -185,7 +185,8 @@ private: /// relocation. bool evaluateFixup(const MCAsmLayout &Layout, const MCFixup &Fixup, const MCFragment *DF, MCValue &Target, - uint64_t &Value, bool &WasForced) const; + const MCSubtargetInfo *STI, uint64_t &Value, + bool &WasForced) const; /// Check whether a fixup can be satisfied, or whether it needs to be relaxed /// (increased in size, in order to hold its value correctly). @@ -221,8 +222,10 @@ private: /// finishLayout - Finalize a layout, including fragment lowering. void finishLayout(MCAsmLayout &Layout); - std::tuple - handleFixup(const MCAsmLayout &Layout, MCFragment &F, const MCFixup &Fixup); + std::tuple handleFixup(const MCAsmLayout &Layout, + MCFragment &F, + const MCFixup &Fixup, + const MCSubtargetInfo *STI); public: struct Symver { diff --git a/llvm/include/llvm/MC/MCInstrItineraries.h b/llvm/include/llvm/MC/MCInstrItineraries.h index d1c2e788ee81..6b29686d8c4f 100644 --- a/llvm/include/llvm/MC/MCInstrItineraries.h +++ b/llvm/include/llvm/MC/MCInstrItineraries.h @@ -110,8 +110,8 @@ struct InstrItinerary { class InstrItineraryData { public: MCSchedModel SchedModel = - MCSchedModel::GetDefaultSchedModel(); ///< Basic machine properties. - const InstrStage *Stages = nullptr; ///< Array of stages selected + MCSchedModel::Default; ///< Basic machine properties. + const InstrStage *Stages = nullptr; ///< Array of stages selected const unsigned *OperandCycles = nullptr; ///< Array of operand cycles selected const unsigned *Forwardings = nullptr; ///< Array of pipeline forwarding paths const InstrItinerary *Itineraries = diff --git a/llvm/include/llvm/MC/MCSchedule.h b/llvm/include/llvm/MC/MCSchedule.h index 98ebe42cfd13..5a6471ac1c89 100644 --- a/llvm/include/llvm/MC/MCSchedule.h +++ b/llvm/include/llvm/MC/MCSchedule.h @@ -390,7 +390,6 @@ struct MCSchedModel { unsigned WriteResourceIdx = 0); /// Returns the default initialized model. - static const MCSchedModel &GetDefaultSchedModel() { return Default; } static const MCSchedModel Default; }; diff --git a/llvm/include/llvm/MC/MCSymbolXCOFF.h b/llvm/include/llvm/MC/MCSymbolXCOFF.h index ef14b0b5c243..11c3b8831ba5 100644 --- a/llvm/include/llvm/MC/MCSymbolXCOFF.h +++ b/llvm/include/llvm/MC/MCSymbolXCOFF.h @@ -17,6 +17,9 @@ namespace llvm { class MCSectionXCOFF; class MCSymbolXCOFF : public MCSymbol { + + enum XCOFFSymbolFlags : uint16_t { SF_EHInfo = 0x0001 }; + public: MCSymbolXCOFF(const StringMapEntry *Name, bool isTemporary) : MCSymbol(SymbolKindXCOFF, Name, isTemporary) {} @@ -65,6 +68,10 @@ public: return getUnqualifiedName(); } + bool isEHInfo() const { return getFlags() & SF_EHInfo; } + + void setEHInfo() const { modifyFlags(SF_EHInfo, SF_EHInfo); } + private: std::optional StorageClass; MCSectionXCOFF *RepresentedCsect = nullptr; diff --git a/llvm/include/llvm/Object/XCOFFObjectFile.h b/llvm/include/llvm/Object/XCOFFObjectFile.h index 9492284ea93d..47926ab3ae40 100644 --- a/llvm/include/llvm/Object/XCOFFObjectFile.h +++ b/llvm/include/llvm/Object/XCOFFObjectFile.h @@ -411,13 +411,13 @@ public: return Entry64->AuxType; } -private: uint8_t getSymbolAlignmentAndType() const { return GETVALUE(SymbolAlignmentAndType); } #undef GETVALUE +private: const XCOFFCsectAuxEnt32 *Entry32 = nullptr; const XCOFFCsectAuxEnt64 *Entry64 = nullptr; }; diff --git a/llvm/include/llvm/Support/AMDGPUAddrSpace.h b/llvm/include/llvm/Support/AMDGPUAddrSpace.h new file mode 100644 index 000000000000..72caf8d458c7 --- /dev/null +++ b/llvm/include/llvm/Support/AMDGPUAddrSpace.h @@ -0,0 +1,83 @@ +//===---------------- AMDGPUAddrSpace.h -------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +/// \file +/// AMDGPU address space definition +/// +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_SUPPORT_AMDGPUADDRSPACE_H +#define LLVM_SUPPORT_AMDGPUADDRSPACE_H + +namespace llvm { +/// OpenCL uses address spaces to differentiate between +/// various memory regions on the hardware. On the CPU +/// all of the address spaces point to the same memory, +/// however on the GPU, each address space points to +/// a separate piece of memory that is unique from other +/// memory locations. +namespace AMDGPUAS { +enum : unsigned { + // The maximum value for flat, generic, local, private, constant and region. + MAX_AMDGPU_ADDRESS = 8, + + FLAT_ADDRESS = 0, ///< Address space for flat memory. + GLOBAL_ADDRESS = 1, ///< Address space for global memory (RAT0, VTX0). + REGION_ADDRESS = 2, ///< Address space for region memory. (GDS) + + CONSTANT_ADDRESS = 4, ///< Address space for constant memory (VTX2). + LOCAL_ADDRESS = 3, ///< Address space for local memory. + PRIVATE_ADDRESS = 5, ///< Address space for private memory. + + CONSTANT_ADDRESS_32BIT = 6, ///< Address space for 32-bit constant memory. + + BUFFER_FAT_POINTER = 7, ///< Address space for 160-bit buffer fat pointers. + ///< Not used in backend. + + BUFFER_RESOURCE = 8, ///< Address space for 128-bit buffer resources. + + /// Internal address spaces. Can be freely renumbered. + STREAMOUT_REGISTER = 128, ///< Address space for GS NGG Streamout registers. + /// end Internal address spaces. + + /// Address space for direct addressable parameter memory (CONST0). + PARAM_D_ADDRESS = 6, + /// Address space for indirect addressable parameter memory (VTX1). + PARAM_I_ADDRESS = 7, + + // Do not re-order the CONSTANT_BUFFER_* enums. Several places depend on + // this order to be able to dynamically index a constant buffer, for + // example: + // + // ConstantBufferAS = CONSTANT_BUFFER_0 + CBIdx + + CONSTANT_BUFFER_0 = 8, + CONSTANT_BUFFER_1 = 9, + CONSTANT_BUFFER_2 = 10, + CONSTANT_BUFFER_3 = 11, + CONSTANT_BUFFER_4 = 12, + CONSTANT_BUFFER_5 = 13, + CONSTANT_BUFFER_6 = 14, + CONSTANT_BUFFER_7 = 15, + CONSTANT_BUFFER_8 = 16, + CONSTANT_BUFFER_9 = 17, + CONSTANT_BUFFER_10 = 18, + CONSTANT_BUFFER_11 = 19, + CONSTANT_BUFFER_12 = 20, + CONSTANT_BUFFER_13 = 21, + CONSTANT_BUFFER_14 = 22, + CONSTANT_BUFFER_15 = 23, + + // Some places use this if the address space can't be determined. + UNKNOWN_ADDRESS_SPACE = ~0u, +}; +} // end namespace AMDGPUAS +} // end namespace llvm + +#endif // LLVM_SUPPORT_AMDGPUADDRSPACE_H diff --git a/llvm/include/llvm/Support/LLVMDriver.h b/llvm/include/llvm/Support/LLVMDriver.h index 1c68f5070777..0b2e265d50b4 100644 --- a/llvm/include/llvm/Support/LLVMDriver.h +++ b/llvm/include/llvm/Support/LLVMDriver.h @@ -9,8 +9,6 @@ #ifndef LLVM_SUPPORT_LLVMDRIVER_H #define LLVM_SUPPORT_LLVMDRIVER_H -#include "llvm/ADT/SmallVector.h" - namespace llvm { struct ToolContext { diff --git a/llvm/include/llvm/Support/TargetOpcodes.def b/llvm/include/llvm/Support/TargetOpcodes.def index 941c6d5f8cad..3824b1c66951 100644 --- a/llvm/include/llvm/Support/TargetOpcodes.def +++ b/llvm/include/llvm/Support/TargetOpcodes.def @@ -415,6 +415,9 @@ HANDLE_TARGET_OPCODE_MARKER(GENERIC_ATOMICRMW_OP_END, G_ATOMICRMW_UDEC_WRAP) // Generic atomic fence HANDLE_TARGET_OPCODE(G_FENCE) +/// Generic prefetch +HANDLE_TARGET_OPCODE(G_PREFETCH) + /// Generic conditional branch instruction. HANDLE_TARGET_OPCODE(G_BRCOND) @@ -454,7 +457,7 @@ HANDLE_TARGET_OPCODE(G_FCONSTANT) /// Generic va_start instruction. Stores to its one pointer operand. HANDLE_TARGET_OPCODE(G_VASTART) -/// Generic va_start instruction. Stores to its one pointer operand. +/// Generic va_arg instruction. Stores to its one pointer operand. HANDLE_TARGET_OPCODE(G_VAARG) // Generic sign extend diff --git a/llvm/include/llvm/Support/TypeSize.h b/llvm/include/llvm/Support/TypeSize.h index 4d9d4f400e92..b00ebf9e8c45 100644 --- a/llvm/include/llvm/Support/TypeSize.h +++ b/llvm/include/llvm/Support/TypeSize.h @@ -15,7 +15,6 @@ #ifndef LLVM_SUPPORT_TYPESIZE_H #define LLVM_SUPPORT_TYPESIZE_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/Support/MathExtras.h" #include "llvm/Support/raw_ostream.h" diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td index 9a9c09d3c20d..73e38b15bf67 100644 --- a/llvm/include/llvm/Target/GenericOpcodes.td +++ b/llvm/include/llvm/Target/GenericOpcodes.td @@ -1209,6 +1209,15 @@ def G_FENCE : GenericInstruction { let hasSideEffects = true; } +// Generic opcode equivalent to the llvm.prefetch intrinsic. +def G_PREFETCH : GenericInstruction { + let OutOperandList = (outs); + let InOperandList = (ins ptype0:$address, i32imm:$rw, i32imm:$locality, i32imm:$cachetype); + let hasSideEffects = true; + let mayLoad = true; + let mayStore = true; +} + //------------------------------------------------------------------------------ // Variadic ops //------------------------------------------------------------------------------ diff --git a/llvm/include/llvm/TargetParser/AArch64TargetParser.h b/llvm/include/llvm/TargetParser/AArch64TargetParser.h index 17cafd146b0e..56c32fae712c 100644 --- a/llvm/include/llvm/TargetParser/AArch64TargetParser.h +++ b/llvm/include/llvm/TargetParser/AArch64TargetParser.h @@ -536,10 +536,9 @@ inline constexpr CpuInfo CpuInfos[] = { {AArch64::AEK_AES, AArch64::AEK_SHA2, AArch64::AEK_DOTPROD, AArch64::AEK_FP16, AArch64::AEK_PROFILE, AArch64::AEK_RCPC, AArch64::AEK_SSBS}))}, - {"neoverse-n2", ARMV8_5A, + {"neoverse-n2", ARMV9A, (AArch64::ExtensionBitset( - {AArch64::AEK_AES, AArch64::AEK_SHA2, AArch64::AEK_SHA3, - AArch64::AEK_SM4, AArch64::AEK_BF16, AArch64::AEK_DOTPROD, + {AArch64::AEK_BF16, AArch64::AEK_DOTPROD, AArch64::AEK_FP16, AArch64::AEK_I8MM, AArch64::AEK_MTE, AArch64::AEK_SB, AArch64::AEK_SSBS, AArch64::AEK_SVE, AArch64::AEK_SVE2, AArch64::AEK_SVE2BITPERM}))}, diff --git a/llvm/include/llvm/TargetParser/ARMTargetParser.def b/llvm/include/llvm/TargetParser/ARMTargetParser.def index 546f6bef34a4..c520ab898cb9 100644 --- a/llvm/include/llvm/TargetParser/ARMTargetParser.def +++ b/llvm/include/llvm/TargetParser/ARMTargetParser.def @@ -308,6 +308,9 @@ ARM_CPU_NAME("cortex-m55", ARMV8_1MMainline, FK_FP_ARMV8_FULLFP16_D16, false, ARM_CPU_NAME("cortex-m85", ARMV8_1MMainline, FK_FP_ARMV8_FULLFP16_D16, false, (ARM::AEK_DSP | ARM::AEK_SIMD | ARM::AEK_FP | ARM::AEK_FP16 | ARM::AEK_RAS | ARM::AEK_PACBTI)) +ARM_CPU_NAME("cortex-m52", ARMV8_1MMainline, FK_FP_ARMV8_FULLFP16_D16, false, + (ARM::AEK_DSP | ARM::AEK_SIMD | ARM::AEK_FP | ARM::AEK_FP16 | + ARM::AEK_RAS | ARM::AEK_PACBTI)) ARM_CPU_NAME("cortex-a32", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("cortex-a35", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) ARM_CPU_NAME("cortex-a53", ARMV8A, FK_CRYPTO_NEON_FP_ARMV8, false, ARM::AEK_CRC) @@ -337,7 +340,7 @@ ARM_CPU_NAME("cortex-x1c", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) ARM_CPU_NAME("neoverse-n1", ARMV8_2A, FK_CRYPTO_NEON_FP_ARMV8, false, (ARM::AEK_FP16 | ARM::AEK_DOTPROD)) -ARM_CPU_NAME("neoverse-n2", ARMV8_5A, FK_CRYPTO_NEON_FP_ARMV8, false, +ARM_CPU_NAME("neoverse-n2", ARMV9A, FK_NEON_FP_ARMV8, false, (ARM::AEK_BF16 | ARM::AEK_DOTPROD | ARM::AEK_I8MM | ARM::AEK_RAS | ARM::AEK_SB)) ARM_CPU_NAME("neoverse-v1", ARMV8_4A, FK_CRYPTO_NEON_FP_ARMV8, false, diff --git a/llvm/include/llvm/TextAPI/RecordsSlice.h b/llvm/include/llvm/TextAPI/RecordsSlice.h index 779321c96efd..8d733fd797ec 100644 --- a/llvm/include/llvm/TextAPI/RecordsSlice.h +++ b/llvm/include/llvm/TextAPI/RecordsSlice.h @@ -128,7 +128,7 @@ public: bool hasBinaryAttrs() const { return BA.get(); } // Determine if record slice is unassigned. - bool isEmpty() const { + bool empty() const { return !hasBinaryAttrs() && Globals.empty() && Classes.empty() && Categories.empty(); } @@ -141,7 +141,7 @@ public: StringRef InstallName; StringRef UUID; StringRef Path; - FileType fileType = FileType::Invalid; + FileType File = FileType::Invalid; llvm::MachO::PackedVersion CurrentVersion; llvm::MachO::PackedVersion CompatVersion; uint8_t SwiftABI = 0; @@ -153,6 +153,10 @@ public: /// Return reference to BinaryAttrs. BinaryAttrs &getBinaryAttrs(); + /// Store any strings owned by RecordSlice into allocator and return back + /// reference to that. + StringRef copyString(StringRef String); + private: const llvm::Triple TargetTriple; // Hold tapi converted triple to avoid unecessary casts. @@ -160,7 +164,6 @@ private: /// BumpPtrAllocator to store generated/copied strings. llvm::BumpPtrAllocator StringAllocator; - StringRef copyString(StringRef String); /// Promote linkage of requested record. It is no-op if linkage type is lower /// than the current assignment. diff --git a/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h b/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h index 9df093d8d5d5..5ff38bdf0481 100644 --- a/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h +++ b/llvm/include/llvm/Transforms/HipStdPar/HipStdPar.h @@ -23,7 +23,6 @@ namespace llvm { class Module; -class ModuleAnaysisManager; class HipStdParAcceleratorCodeSelectionPass : public PassInfoMixin { diff --git a/llvm/include/llvm/Transforms/IPO/BlockExtractor.h b/llvm/include/llvm/Transforms/IPO/BlockExtractor.h index 6211027bd672..cf6b1666b4fc 100644 --- a/llvm/include/llvm/Transforms/IPO/BlockExtractor.h +++ b/llvm/include/llvm/Transforms/IPO/BlockExtractor.h @@ -16,7 +16,6 @@ #include -#include "llvm/ADT/SmallVector.h" #include "llvm/IR/PassManager.h" namespace llvm { diff --git a/llvm/include/llvm/Transforms/IPO/EmbedBitcodePass.h b/llvm/include/llvm/Transforms/IPO/EmbedBitcodePass.h index c35048c91aba..2bb7d5f1fcf1 100644 --- a/llvm/include/llvm/Transforms/IPO/EmbedBitcodePass.h +++ b/llvm/include/llvm/Transforms/IPO/EmbedBitcodePass.h @@ -22,7 +22,6 @@ namespace llvm { class Module; -class ModulePass; class Pass; /// Pass embeds a copy of the module optimized with the provided pass pipeline diff --git a/llvm/include/llvm/Transforms/Instrumentation.h b/llvm/include/llvm/Transforms/Instrumentation.h index 3035cdeeacf6..ea97ab2562a5 100644 --- a/llvm/include/llvm/Transforms/Instrumentation.h +++ b/llvm/include/llvm/Transforms/Instrumentation.h @@ -49,6 +49,12 @@ GlobalVariable *createPrivateGlobalForString(Module &M, StringRef Str, // Returns nullptr on failure. Comdat *getOrCreateFunctionComdat(Function &F, Triple &T); +// Place global in a large section for x86-64 ELF binaries to mitigate +// relocation overflow pressure. This can be be used for metadata globals that +// aren't directly accessed by code, which has no performance impact. +void setGlobalVariableLargeSection(const Triple &TargetTriple, + GlobalVariable &GV); + // Insert GCOV profiling instrumentation struct GCOVOptions { static GCOVOptions getDefault(); diff --git a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h index c106e1651e80..95eb3019eab0 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h +++ b/llvm/include/llvm/Transforms/Instrumentation/InstrProfiling.h @@ -31,31 +31,46 @@ using LoadStorePair = std::pair; /// Instrumentation based profiling lowering pass. This pass lowers /// the profile instrumented code generated by FE or the IR based /// instrumentation pass. -class InstrProfiling : public PassInfoMixin { +class InstrProfilingLoweringPass + : public PassInfoMixin { + const InstrProfOptions Options = {}; + // Is this lowering for the context-sensitive instrumentation. + const bool IsCS = false; + public: - InstrProfiling() : IsCS(false) {} - InstrProfiling(const InstrProfOptions &Options, bool IsCS = false) + InstrProfilingLoweringPass() = default; + InstrProfilingLoweringPass(const InstrProfOptions &Options, bool IsCS = false) : Options(Options), IsCS(IsCS) {} PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM); - bool run(Module &M, - std::function GetTLI); +}; + +class InstrProfiling final { +public: + InstrProfiling(Module &M, const InstrProfOptions &Options, + std::function GetTLI, + bool IsCS) + : M(M), Options(Options), TT(Triple(M.getTargetTriple())), IsCS(IsCS), + GetTLI(GetTLI) {} + + bool lower(); private: - InstrProfOptions Options; - Module *M; - Triple TT; + Module &M; + const InstrProfOptions Options; + const Triple TT; + // Is this lowering for the context-sensitive instrumentation. + const bool IsCS; + std::function GetTLI; struct PerFunctionProfileData { - uint32_t NumValueSites[IPVK_Last + 1]; + uint32_t NumValueSites[IPVK_Last + 1] = {}; GlobalVariable *RegionCounters = nullptr; GlobalVariable *DataVar = nullptr; GlobalVariable *RegionBitmaps = nullptr; uint32_t NumBitmapBytes = 0; - PerFunctionProfileData() { - memset(NumValueSites, 0, sizeof(uint32_t) * (IPVK_Last + 1)); - } + PerFunctionProfileData() = default; }; DenseMap ProfileDataMap; /// If runtime relocation is enabled, this maps functions to the load @@ -64,11 +79,8 @@ private: std::vector CompilerUsedVars; std::vector UsedVars; std::vector ReferencedNames; - GlobalVariable *NamesVar; - size_t NamesSize; - - // Is this lowering for the context-sensitive instrumentation. - bool IsCS; + GlobalVariable *NamesVar = nullptr; + size_t NamesSize = 0; // vector of counter load/store pairs to be register promoted. std::vector PromotionCandidates; diff --git a/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h b/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h index 293133b29cd9..f92c6b4775a2 100644 --- a/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h +++ b/llvm/include/llvm/Transforms/Instrumentation/MemProfiler.h @@ -17,9 +17,7 @@ namespace llvm { class Function; -class FunctionPass; class Module; -class ModulePass; namespace vfs { class FileSystem; diff --git a/llvm/include/llvm/Transforms/Scalar/Reassociate.h b/llvm/include/llvm/Transforms/Scalar/Reassociate.h index 28794d27325a..7e47f8ae5d81 100644 --- a/llvm/include/llvm/Transforms/Scalar/Reassociate.h +++ b/llvm/include/llvm/Transforms/Scalar/Reassociate.h @@ -102,7 +102,8 @@ private: void canonicalizeOperands(Instruction *I); void ReassociateExpression(BinaryOperator *I); void RewriteExprTree(BinaryOperator *I, - SmallVectorImpl &Ops); + SmallVectorImpl &Ops, + bool HasNUW); Value *OptimizeExpression(BinaryOperator *I, SmallVectorImpl &Ops); Value *OptimizeAdd(Instruction *I, diff --git a/llvm/include/llvm/Transforms/Scalar/Scalarizer.h b/llvm/include/llvm/Transforms/Scalar/Scalarizer.h index c2d9151b4971..45e25cbf2821 100644 --- a/llvm/include/llvm/Transforms/Scalar/Scalarizer.h +++ b/llvm/include/llvm/Transforms/Scalar/Scalarizer.h @@ -24,7 +24,6 @@ namespace llvm { class Function; -class FunctionPass; struct ScalarizerPassOptions { // These options correspond 1:1 to cl::opt options defined in diff --git a/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h b/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h index 417e9668527b..b97ee23fc0e6 100644 --- a/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h +++ b/llvm/include/llvm/Transforms/Scalar/SimpleLoopUnswitch.h @@ -17,7 +17,6 @@ namespace llvm { class LPMUpdater; class Loop; -class Pass; class StringRef; class raw_ostream; diff --git a/llvm/include/llvm/Transforms/Utils/ValueMapper.h b/llvm/include/llvm/Transforms/Utils/ValueMapper.h index eedd25f898c0..e1f2796d97ce 100644 --- a/llvm/include/llvm/Transforms/Utils/ValueMapper.h +++ b/llvm/include/llvm/Transforms/Utils/ValueMapper.h @@ -22,7 +22,6 @@ namespace llvm { class Constant; -class DIBuilder; class DPValue; class Function; class GlobalVariable; diff --git a/llvm/lib/Analysis/AliasSetTracker.cpp b/llvm/lib/Analysis/AliasSetTracker.cpp index 91b889116dfa..debdd328ce53 100644 --- a/llvm/lib/Analysis/AliasSetTracker.cpp +++ b/llvm/lib/Analysis/AliasSetTracker.cpp @@ -348,8 +348,16 @@ AliasSet &AliasSetTracker::getAliasSetFor(const MemoryLocation &MemLoc) { // due to a quirk of alias analysis behavior. Since alias(undef, undef) // is NoAlias, mergeAliasSetsForPointer(undef, ...) will not find the // the right set for undef, even if it exists. - if (Entry.updateSizeAndAAInfo(Size, AAInfo)) + if (Entry.updateSizeAndAAInfo(Size, AAInfo)) { mergeAliasSetsForPointer(Pointer, Size, AAInfo, MustAliasAll); + + // For MustAlias sets, also update Size/AAInfo of the representative + // pointer. + AliasSet &AS = *Entry.getAliasSet(*this); + if (AS.isMustAlias()) + if (AliasSet::PointerRec *P = AS.getSomePointer()) + P->updateSizeAndAAInfo(Size, AAInfo); + } // Return the set! return *Entry.getAliasSet(*this)->getForwardedTarget(*this); } diff --git a/llvm/lib/Analysis/DomConditionCache.cpp b/llvm/lib/Analysis/DomConditionCache.cpp index 351881fe9e1f..c7f4cab41588 100644 --- a/llvm/lib/Analysis/DomConditionCache.cpp +++ b/llvm/lib/Analysis/DomConditionCache.cpp @@ -36,8 +36,7 @@ static void findAffectedValues(Value *Cond, ICmpInst::Predicate Pred; Value *A; - Constant *C; - if (match(Cond, m_ICmp(Pred, m_Value(A), m_Constant(C)))) { + if (match(Cond, m_ICmp(Pred, m_Value(A), m_Constant()))) { AddAffected(A); if (ICmpInst::isEquality(Pred)) { diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp index 46629e381bc3..1aa324c6b5f3 100644 --- a/llvm/lib/Analysis/IVDescriptors.cpp +++ b/llvm/lib/Analysis/IVDescriptors.cpp @@ -24,8 +24,6 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/KnownBits.h" -#include - using namespace llvm; using namespace llvm::PatternMatch; diff --git a/llvm/lib/Analysis/ValueTracking.cpp b/llvm/lib/Analysis/ValueTracking.cpp index ee4f97f3bf5e..5445746ab2a1 100644 --- a/llvm/lib/Analysis/ValueTracking.cpp +++ b/llvm/lib/Analysis/ValueTracking.cpp @@ -616,11 +616,11 @@ static bool isKnownNonZeroFromAssume(const Value *V, const SimplifyQuery &Q) { static void computeKnownBitsFromCmp(const Value *V, CmpInst::Predicate Pred, Value *LHS, Value *RHS, KnownBits &Known, - unsigned Depth, const SimplifyQuery &Q) { + const SimplifyQuery &Q) { if (RHS->getType()->isPointerTy()) { // Handle comparison of pointer to null explicitly, as it will not be // covered by the m_APInt() logic below. - if (match(RHS, m_Zero())) { + if (LHS == V && match(RHS, m_Zero())) { switch (Pred) { case ICmpInst::ICMP_EQ: Known.setAllZero(); @@ -720,13 +720,13 @@ void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known, BasicBlockEdge Edge0(BI->getParent(), BI->getSuccessor(0)); if (Q.DT->dominates(Edge0, Q.CxtI->getParent())) computeKnownBitsFromCmp(V, Cmp->getPredicate(), Cmp->getOperand(0), - Cmp->getOperand(1), Known, Depth, Q); + Cmp->getOperand(1), Known, Q); BasicBlockEdge Edge1(BI->getParent(), BI->getSuccessor(1)); if (Q.DT->dominates(Edge1, Q.CxtI->getParent())) computeKnownBitsFromCmp(V, Cmp->getInversePredicate(), Cmp->getOperand(0), Cmp->getOperand(1), Known, - Depth, Q); + Q); } if (Known.hasConflict()) @@ -794,7 +794,7 @@ void llvm::computeKnownBitsFromContext(const Value *V, KnownBits &Known, continue; computeKnownBitsFromCmp(V, Cmp->getPredicate(), Cmp->getOperand(0), - Cmp->getOperand(1), Known, Depth, Q); + Cmp->getOperand(1), Known, Q); } // Conflicting assumption: Undefined behavior will occur on this execution @@ -1465,8 +1465,10 @@ static void computeKnownBitsFromOperator(const Operator *I, Q.IIQ.getMetadata(cast(I), LLVMContext::MD_range)) computeKnownBitsFromRangeMetadata(*MD, Known); if (const Value *RV = cast(I)->getReturnedArgOperand()) { - computeKnownBits(RV, Known2, Depth + 1, Q); - Known = Known.unionWith(Known2); + if (RV->getType() == I->getType()) { + computeKnownBits(RV, Known2, Depth + 1, Q); + Known = Known.unionWith(Known2); + } } if (const IntrinsicInst *II = dyn_cast(I)) { switch (II->getIntrinsicID()) { @@ -2712,7 +2714,7 @@ static bool isKnownNonZeroFromOperator(const Operator *I, if (const auto *RP = getArgumentAliasingToReturnedPointer(Call, true)) return isKnownNonZero(RP, Depth, Q); } else if (const Value *RV = cast(I)->getReturnedArgOperand()) { - if (isKnownNonZero(RV, Depth, Q)) + if (RV->getType() == I->getType() && isKnownNonZero(RV, Depth, Q)) return true; } @@ -8024,7 +8026,7 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, for (unsigned i = 0; i != 2; ++i) { Value *L = P->getIncomingValue(i); Value *R = P->getIncomingValue(!i); - Operator *LU = dyn_cast(L); + auto *LU = dyn_cast(L); if (!LU) continue; unsigned Opcode = LU->getOpcode(); @@ -8062,7 +8064,7 @@ bool llvm::matchSimpleRecurrence(const PHINode *P, BinaryOperator *&BO, // OR // %iv = [R, %entry], [%iv.next, %backedge] // %iv.next = binop L, %iv - BO = cast(LU); + BO = LU; Start = R; Step = L; return true; diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp index bf01b39e6f97..919c69fe2783 100644 --- a/llvm/lib/AsmParser/LLLexer.cpp +++ b/llvm/lib/AsmParser/LLLexer.cpp @@ -698,6 +698,7 @@ lltok::Kind LLLexer::LexIdentifier() { KEYWORD(uinc_wrap); KEYWORD(udec_wrap); + KEYWORD(splat); KEYWORD(vscale); KEYWORD(x); KEYWORD(blockaddress); diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp index 5aa00225170b..fb9e1ba875e1 100644 --- a/llvm/lib/AsmParser/LLParser.cpp +++ b/llvm/lib/AsmParser/LLParser.cpp @@ -3981,6 +3981,21 @@ bool LLParser::parseValID(ValID &ID, PerFunctionState *PFS, Type *ExpectedTy) { return false; } + case lltok::kw_splat: { + Lex.Lex(); + if (parseToken(lltok::lparen, "expected '(' after vector splat")) + return true; + Constant *C; + if (parseGlobalTypeAndValue(C)) + return true; + if (parseToken(lltok::rparen, "expected ')' at end of vector splat")) + return true; + + ID.ConstantVal = C; + ID.Kind = ValID::t_ConstantSplat; + return false; + } + case lltok::kw_getelementptr: case lltok::kw_shufflevector: case lltok::kw_insertelement: @@ -5824,6 +5839,17 @@ bool LLParser::convertValIDToValue(Type *Ty, ValID &ID, Value *&V, "' but expected '" + getTypeString(Ty) + "'"); V = ID.ConstantVal; return false; + case ValID::t_ConstantSplat: + if (!Ty->isVectorTy()) + return error(ID.Loc, "vector constant must have vector type"); + if (ID.ConstantVal->getType() != Ty->getScalarType()) + return error(ID.Loc, "constant expression type mismatch: got type '" + + getTypeString(ID.ConstantVal->getType()) + + "' but expected '" + + getTypeString(Ty->getScalarType()) + "'"); + V = ConstantVector::getSplat(cast(Ty)->getElementCount(), + ID.ConstantVal); + return false; case ValID::t_ConstantStruct: case ValID::t_PackedConstantStruct: if (StructType *ST = dyn_cast(Ty)) { @@ -5861,6 +5887,7 @@ bool LLParser::parseConstantValue(Type *Ty, Constant *&C) { case ValID::t_APFloat: case ValID::t_Undef: case ValID::t_Constant: + case ValID::t_ConstantSplat: case ValID::t_ConstantStruct: case ValID::t_PackedConstantStruct: { Value *V; @@ -8622,9 +8649,9 @@ static void resolveFwdRef(ValueInfo *Fwd, ValueInfo &Resolved) { /// Stores the given Name/GUID and associated summary into the Index. /// Also updates any forward references to the associated entry ID. -void LLParser::addGlobalValueToIndex( +bool LLParser::addGlobalValueToIndex( std::string Name, GlobalValue::GUID GUID, GlobalValue::LinkageTypes Linkage, - unsigned ID, std::unique_ptr Summary) { + unsigned ID, std::unique_ptr Summary, LocTy Loc) { // First create the ValueInfo utilizing the Name or GUID. ValueInfo VI; if (GUID != 0) { @@ -8634,7 +8661,9 @@ void LLParser::addGlobalValueToIndex( assert(!Name.empty()); if (M) { auto *GV = M->getNamedValue(Name); - assert(GV); + if (!GV) + return error(Loc, "Reference to undefined global \"" + Name + "\""); + VI = Index->getOrInsertValueInfo(GV); } else { assert( @@ -8682,6 +8711,8 @@ void LLParser::addGlobalValueToIndex( NumberedValueInfos.resize(ID + 1); NumberedValueInfos[ID] = VI; } + + return false; } /// parseSummaryIndexFlags @@ -8728,6 +8759,7 @@ bool LLParser::parseGVEntry(unsigned ID) { parseToken(lltok::lparen, "expected '(' here")) return true; + LocTy Loc = Lex.getLoc(); std::string Name; GlobalValue::GUID GUID = 0; switch (Lex.getKind()) { @@ -8757,9 +8789,8 @@ bool LLParser::parseGVEntry(unsigned ID) { // an external definition. We pass ExternalLinkage since that is only // used when the GUID must be computed from Name, and in that case // the symbol must have external linkage. - addGlobalValueToIndex(Name, GUID, GlobalValue::ExternalLinkage, ID, - nullptr); - return false; + return addGlobalValueToIndex(Name, GUID, GlobalValue::ExternalLinkage, ID, + nullptr, Loc); } // Have a list of summaries @@ -8800,6 +8831,7 @@ bool LLParser::parseGVEntry(unsigned ID) { /// [',' OptionalRefs]? ')' bool LLParser::parseFunctionSummary(std::string Name, GlobalValue::GUID GUID, unsigned ID) { + LocTy Loc = Lex.getLoc(); assert(Lex.getKind() == lltok::kw_function); Lex.Lex(); @@ -8876,10 +8908,9 @@ bool LLParser::parseFunctionSummary(std::string Name, GlobalValue::GUID GUID, FS->setModulePath(ModulePath); - addGlobalValueToIndex(Name, GUID, (GlobalValue::LinkageTypes)GVFlags.Linkage, - ID, std::move(FS)); - - return false; + return addGlobalValueToIndex(Name, GUID, + (GlobalValue::LinkageTypes)GVFlags.Linkage, ID, + std::move(FS), Loc); } /// VariableSummary @@ -8887,6 +8918,7 @@ bool LLParser::parseFunctionSummary(std::string Name, GlobalValue::GUID GUID, /// [',' OptionalRefs]? ')' bool LLParser::parseVariableSummary(std::string Name, GlobalValue::GUID GUID, unsigned ID) { + LocTy Loc = Lex.getLoc(); assert(Lex.getKind() == lltok::kw_variable); Lex.Lex(); @@ -8934,10 +8966,9 @@ bool LLParser::parseVariableSummary(std::string Name, GlobalValue::GUID GUID, GS->setModulePath(ModulePath); GS->setVTableFuncs(std::move(VTableFuncs)); - addGlobalValueToIndex(Name, GUID, (GlobalValue::LinkageTypes)GVFlags.Linkage, - ID, std::move(GS)); - - return false; + return addGlobalValueToIndex(Name, GUID, + (GlobalValue::LinkageTypes)GVFlags.Linkage, ID, + std::move(GS), Loc); } /// AliasSummary @@ -8984,10 +9015,9 @@ bool LLParser::parseAliasSummary(std::string Name, GlobalValue::GUID GUID, AS->setAliasee(AliaseeVI, Summary); } - addGlobalValueToIndex(Name, GUID, (GlobalValue::LinkageTypes)GVFlags.Linkage, - ID, std::move(AS)); - - return false; + return addGlobalValueToIndex(Name, GUID, + (GlobalValue::LinkageTypes)GVFlags.Linkage, ID, + std::move(AS), Loc); } /// Flag diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 9d7e838733b5..a11690a19a0e 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -5248,7 +5248,7 @@ Error BitcodeReader::parseFunctionBody(Function *F) { return error( "Invalid record: operand number exceeded available operands"); - unsigned PredVal = Record[OpNum]; + CmpInst::Predicate PredVal = CmpInst::Predicate(Record[OpNum]); bool IsFP = LHS->getType()->isFPOrFPVectorTy(); FastMathFlags FMF; if (IsFP && Record.size() > OpNum+1) @@ -5257,10 +5257,15 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (OpNum+1 != Record.size()) return error("Invalid record"); - if (LHS->getType()->isFPOrFPVectorTy()) - I = new FCmpInst((FCmpInst::Predicate)PredVal, LHS, RHS); - else - I = new ICmpInst((ICmpInst::Predicate)PredVal, LHS, RHS); + if (IsFP) { + if (!CmpInst::isFPPredicate(PredVal)) + return error("Invalid fcmp predicate"); + I = new FCmpInst(PredVal, LHS, RHS); + } else { + if (!CmpInst::isIntPredicate(PredVal)) + return error("Invalid icmp predicate"); + I = new ICmpInst(PredVal, LHS, RHS); + } ResTypeID = getVirtualTypeID(I->getType()->getScalarType()); if (LHS->getType()->isVectorTy()) @@ -5363,6 +5368,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { Type *TokenTy = Type::getTokenTy(Context); Value *ParentPad = getValue(Record, Idx++, NextValueNo, TokenTy, getVirtualTypeID(TokenTy), CurBB); + if (!ParentPad) + return error("Invalid record"); unsigned NumHandlers = Record[Idx++]; @@ -5404,6 +5411,8 @@ Error BitcodeReader::parseFunctionBody(Function *F) { Type *TokenTy = Type::getTokenTy(Context); Value *ParentPad = getValue(Record, Idx++, NextValueNo, TokenTy, getVirtualTypeID(TokenTy), CurBB); + if (!ParentPad) + return error("Invald record"); unsigned NumArgOperands = Record[Idx++]; @@ -5958,6 +5967,9 @@ Error BitcodeReader::parseFunctionBody(Function *F) { if (!Align) Align = DL.getPrefTypeAlign(Ty); + if (!Size->getType()->isIntegerTy()) + return error("alloca element count must have integer type"); + AllocaInst *AI = new AllocaInst(Ty, AS, Size, *Align); AI->setUsedWithInAlloca(InAlloca); AI->setSwiftError(SwiftError); diff --git a/llvm/lib/Bitcode/Reader/ValueList.cpp b/llvm/lib/Bitcode/Reader/ValueList.cpp index b9dbf904c89e..f5568a923b11 100644 --- a/llvm/lib/Bitcode/Reader/ValueList.cpp +++ b/llvm/lib/Bitcode/Reader/ValueList.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "ValueList.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/IR/Argument.h" #include "llvm/IR/Constant.h" #include "llvm/IR/Constants.h" diff --git a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp index f00528023c91..ad3ad9928987 100644 --- a/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp +++ b/llvm/lib/CodeGen/AssignmentTrackingAnalysis.cpp @@ -2269,14 +2269,14 @@ static bool removeRedundantDbgLocsUsingBackwardScan(const BasicBlock *BB, FunctionVarLocsBuilder &FnVarLocs) { bool Changed = false; - SmallDenseMap VariableDefinedBits; + SmallDenseMap VariableDefinedBytes; // Scan over the entire block, not just over the instructions mapped by // FnVarLocs, because wedges in FnVarLocs may only be seperated by debug // instructions. for (const Instruction &I : reverse(*BB)) { if (!isa(I)) { // Sequence of consecutive defs ended. Clear map for the next one. - VariableDefinedBits.clear(); + VariableDefinedBytes.clear(); } // Get the location defs that start just before this instruction. @@ -2295,9 +2295,15 @@ removeRedundantDbgLocsUsingBackwardScan(const BasicBlock *BB, DebugAggregate Aggr = getAggregate(FnVarLocs.getVariable(RIt->VariableID)); uint64_t SizeInBits = Aggr.first->getSizeInBits().value_or(0); + uint64_t SizeInBytes = divideCeil(SizeInBits, 8); - if (SizeInBits == 0) { + // Cutoff for large variables to prevent expensive bitvector operations. + const uint64_t MaxSizeBytes = 2048; + + if (SizeInBytes == 0 || SizeInBytes > MaxSizeBytes) { // If the size is unknown (0) then keep this location def to be safe. + // Do the same for defs of large variables, which would be expensive + // to represent with a BitVector. NewDefsReversed.push_back(*RIt); continue; } @@ -2305,23 +2311,24 @@ removeRedundantDbgLocsUsingBackwardScan(const BasicBlock *BB, // Only keep this location definition if it is not fully eclipsed by // other definitions in this wedge that come after it - // Inert the bits the location definition defines. + // Inert the bytes the location definition defines. auto InsertResult = - VariableDefinedBits.try_emplace(Aggr, BitVector(SizeInBits)); + VariableDefinedBytes.try_emplace(Aggr, BitVector(SizeInBytes)); bool FirstDefinition = InsertResult.second; - BitVector &DefinedBits = InsertResult.first->second; + BitVector &DefinedBytes = InsertResult.first->second; DIExpression::FragmentInfo Fragment = RIt->Expr->getFragmentInfo().value_or( DIExpression::FragmentInfo(SizeInBits, 0)); bool InvalidFragment = Fragment.endInBits() > SizeInBits; + uint64_t StartInBytes = Fragment.startInBits() / 8; + uint64_t EndInBytes = divideCeil(Fragment.endInBits(), 8); - // If this defines any previously undefined bits, keep it. + // If this defines any previously undefined bytes, keep it. if (FirstDefinition || InvalidFragment || - DefinedBits.find_first_unset_in(Fragment.startInBits(), - Fragment.endInBits()) != -1) { + DefinedBytes.find_first_unset_in(StartInBytes, EndInBytes) != -1) { if (!InvalidFragment) - DefinedBits.set(Fragment.startInBits(), Fragment.endInBits()); + DefinedBytes.set(StartInBytes, EndInBytes); NewDefsReversed.push_back(*RIt); continue; } diff --git a/llvm/lib/CodeGen/DwarfEHPrepare.cpp b/llvm/lib/CodeGen/DwarfEHPrepare.cpp index a44aaf33c6a4..e7eb34d8e651 100644 --- a/llvm/lib/CodeGen/DwarfEHPrepare.cpp +++ b/llvm/lib/CodeGen/DwarfEHPrepare.cpp @@ -42,7 +42,7 @@ using namespace llvm; -#define DEBUG_TYPE "dwarfehprepare" +#define DEBUG_TYPE "dwarf-eh-prepare" STATISTIC(NumResumesLowered, "Number of resume calls lowered"); STATISTIC(NumCleanupLandingPadsUnreachable, diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 14a4e72152e7..27a53e55f32f 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -2435,6 +2435,21 @@ bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, MIRBuilder.buildInstr(TargetOpcode::G_RESET_FPMODE, {}, {}); return true; } + case Intrinsic::prefetch: { + Value *Addr = CI.getOperand(0); + unsigned RW = cast(CI.getOperand(1))->getZExtValue(); + unsigned Locality = cast(CI.getOperand(2))->getZExtValue(); + unsigned CacheType = cast(CI.getOperand(3))->getZExtValue(); + + auto Flags = RW ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad; + auto &MMO = *MF->getMachineMemOperand(MachinePointerInfo(Addr), Flags, + LLT(), Align()); + + MIRBuilder.buildPrefetch(getOrCreateVReg(*Addr), RW, Locality, CacheType, + MMO); + + return true; + } #define INSTRUCTION(NAME, NARG, ROUND_MODE, INTRINSIC) \ case Intrinsic::INTRINSIC: #include "llvm/IR/ConstrainedOps.def" diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp index 045fc78218da..37e7153be572 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp @@ -3793,6 +3793,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) { return lowerTRUNC(MI); GISEL_VECREDUCE_CASES_NONSEQ return lowerVectorReduction(MI); + case G_VAARG: + return lowerVAArg(MI); } } @@ -7887,6 +7889,56 @@ LegalizerHelper::lowerVectorReduction(MachineInstr &MI) { return UnableToLegalize; } +static Type *getTypeForLLT(LLT Ty, LLVMContext &C); + +LegalizerHelper::LegalizeResult LegalizerHelper::lowerVAArg(MachineInstr &MI) { + MachineFunction &MF = *MI.getMF(); + const DataLayout &DL = MIRBuilder.getDataLayout(); + LLVMContext &Ctx = MF.getFunction().getContext(); + Register ListPtr = MI.getOperand(1).getReg(); + LLT PtrTy = MRI.getType(ListPtr); + + // LstPtr is a pointer to the head of the list. Get the address + // of the head of the list. + Align PtrAlignment = DL.getABITypeAlign(getTypeForLLT(PtrTy, Ctx)); + MachineMemOperand *PtrLoadMMO = MF.getMachineMemOperand( + MachinePointerInfo(), MachineMemOperand::MOLoad, PtrTy, PtrAlignment); + auto VAList = MIRBuilder.buildLoad(PtrTy, ListPtr, *PtrLoadMMO).getReg(0); + + const Align A(MI.getOperand(2).getImm()); + LLT PtrTyAsScalarTy = LLT::scalar(PtrTy.getSizeInBits()); + if (A > TLI.getMinStackArgumentAlignment()) { + Register AlignAmt = + MIRBuilder.buildConstant(PtrTyAsScalarTy, A.value() - 1).getReg(0); + auto AddDst = MIRBuilder.buildPtrAdd(PtrTy, VAList, AlignAmt); + auto AndDst = MIRBuilder.buildMaskLowPtrBits(PtrTy, AddDst, Log2(A)); + VAList = AndDst.getReg(0); + } + + // Increment the pointer, VAList, to the next vaarg + // The list should be bumped by the size of element in the current head of + // list. + Register Dst = MI.getOperand(0).getReg(); + LLT LLTTy = MRI.getType(Dst); + Type *Ty = getTypeForLLT(LLTTy, Ctx); + auto IncAmt = + MIRBuilder.buildConstant(PtrTyAsScalarTy, DL.getTypeAllocSize(Ty)); + auto Succ = MIRBuilder.buildPtrAdd(PtrTy, VAList, IncAmt); + + // Store the increment VAList to the legalized pointer + MachineMemOperand *StoreMMO = MF.getMachineMemOperand( + MachinePointerInfo(), MachineMemOperand::MOStore, PtrTy, PtrAlignment); + MIRBuilder.buildStore(Succ, ListPtr, *StoreMMO); + // Load the actual argument out of the pointer VAList + Align EltAlignment = DL.getABITypeAlign(Ty); + MachineMemOperand *EltLoadMMO = MF.getMachineMemOperand( + MachinePointerInfo(), MachineMemOperand::MOLoad, LLTTy, EltAlignment); + MIRBuilder.buildLoad(Dst, VAList, *EltLoadMMO); + + MI.eraseFromParent(); + return Legalized; +} + static bool shouldLowerMemFuncForSize(const MachineFunction &MF) { // On Darwin, -Os means optimize for size without hurting performance, so // only really optimize for size when -Oz (MinSize) is used. diff --git a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp index 80e9c08e850b..6e1c0e907e8e 100644 --- a/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp +++ b/llvm/lib/CodeGen/GlobalISel/MachineIRBuilder.cpp @@ -1051,6 +1051,18 @@ MachineIRBuilder::buildFence(unsigned Ordering, unsigned Scope) { .addImm(Scope); } +MachineInstrBuilder MachineIRBuilder::buildPrefetch(const SrcOp &Addr, + unsigned RW, + unsigned Locality, + unsigned CacheType, + MachineMemOperand &MMO) { + auto MIB = buildInstr(TargetOpcode::G_PREFETCH); + Addr.addSrcToMIB(MIB); + MIB.addImm(RW).addImm(Locality).addImm(CacheType); + MIB.addMemOperand(&MMO); + return MIB; +} + MachineInstrBuilder MachineIRBuilder::buildBlockAddress(Register Res, const BlockAddress *BA) { #ifndef NDEBUG diff --git a/llvm/lib/CodeGen/InterleavedAccessPass.cpp b/llvm/lib/CodeGen/InterleavedAccessPass.cpp index 65a6859a006a..2a0daf404c97 100644 --- a/llvm/lib/CodeGen/InterleavedAccessPass.cpp +++ b/llvm/lib/CodeGen/InterleavedAccessPass.cpp @@ -48,6 +48,7 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/SetVector.h" #include "llvm/ADT/SmallVector.h" +#include "llvm/CodeGen/InterleavedAccess.h" #include "llvm/CodeGen/TargetLowering.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" @@ -82,22 +83,14 @@ static cl::opt LowerInterleavedAccesses( namespace { -class InterleavedAccess : public FunctionPass { -public: - static char ID; - - InterleavedAccess() : FunctionPass(ID) { - initializeInterleavedAccessPass(*PassRegistry::getPassRegistry()); - } +class InterleavedAccessImpl { + friend class InterleavedAccess; - StringRef getPassName() const override { return "Interleaved Access Pass"; } - - bool runOnFunction(Function &F) override; - - void getAnalysisUsage(AnalysisUsage &AU) const override { - AU.addRequired(); - AU.setPreservesCFG(); - } +public: + InterleavedAccessImpl() = default; + InterleavedAccessImpl(DominatorTree *DT, const TargetLowering *TLI) + : DT(DT), TLI(TLI), MaxFactor(TLI->getMaxSupportedInterleaveFactor()) {} + bool runOnFunction(Function &F); private: DominatorTree *DT = nullptr; @@ -141,10 +134,60 @@ private: LoadInst *LI); }; +class InterleavedAccess : public FunctionPass { + InterleavedAccessImpl Impl; + +public: + static char ID; + + InterleavedAccess() : FunctionPass(ID) { + initializeInterleavedAccessPass(*PassRegistry::getPassRegistry()); + } + + StringRef getPassName() const override { return "Interleaved Access Pass"; } + + bool runOnFunction(Function &F) override; + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.addRequired(); + AU.setPreservesCFG(); + } +}; + } // end anonymous namespace. +PreservedAnalyses InterleavedAccessPass::run(Function &F, + FunctionAnalysisManager &FAM) { + auto *DT = &FAM.getResult(F); + auto *TLI = TM->getSubtargetImpl(F)->getTargetLowering(); + InterleavedAccessImpl Impl(DT, TLI); + bool Changed = Impl.runOnFunction(F); + + if (!Changed) + return PreservedAnalyses::all(); + + PreservedAnalyses PA; + PA.preserveSet(); + return PA; +} + char InterleavedAccess::ID = 0; +bool InterleavedAccess::runOnFunction(Function &F) { + auto *TPC = getAnalysisIfAvailable(); + if (!TPC || !LowerInterleavedAccesses) + return false; + + LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n"); + + Impl.DT = &getAnalysis().getDomTree(); + auto &TM = TPC->getTM(); + Impl.TLI = TM.getSubtargetImpl(F)->getTargetLowering(); + Impl.MaxFactor = Impl.TLI->getMaxSupportedInterleaveFactor(); + + return Impl.runOnFunction(F); +} + INITIALIZE_PASS_BEGIN(InterleavedAccess, DEBUG_TYPE, "Lower interleaved memory accesses to target specific intrinsics", false, false) @@ -228,7 +271,7 @@ static bool isReInterleaveMask(ShuffleVectorInst *SVI, unsigned &Factor, return false; } -bool InterleavedAccess::lowerInterleavedLoad( +bool InterleavedAccessImpl::lowerInterleavedLoad( LoadInst *LI, SmallVector &DeadInsts) { if (!LI->isSimple() || isa(LI->getType())) return false; @@ -334,7 +377,7 @@ bool InterleavedAccess::lowerInterleavedLoad( return true; } -bool InterleavedAccess::replaceBinOpShuffles( +bool InterleavedAccessImpl::replaceBinOpShuffles( ArrayRef BinOpShuffles, SmallVectorImpl &Shuffles, LoadInst *LI) { for (auto *SVI : BinOpShuffles) { @@ -367,7 +410,7 @@ bool InterleavedAccess::replaceBinOpShuffles( return !BinOpShuffles.empty(); } -bool InterleavedAccess::tryReplaceExtracts( +bool InterleavedAccessImpl::tryReplaceExtracts( ArrayRef Extracts, ArrayRef Shuffles) { // If there aren't any extractelement instructions to modify, there's nothing @@ -431,7 +474,7 @@ bool InterleavedAccess::tryReplaceExtracts( return true; } -bool InterleavedAccess::lowerInterleavedStore( +bool InterleavedAccessImpl::lowerInterleavedStore( StoreInst *SI, SmallVector &DeadInsts) { if (!SI->isSimple()) return false; @@ -457,7 +500,7 @@ bool InterleavedAccess::lowerInterleavedStore( return true; } -bool InterleavedAccess::lowerDeinterleaveIntrinsic( +bool InterleavedAccessImpl::lowerDeinterleaveIntrinsic( IntrinsicInst *DI, SmallVector &DeadInsts) { LoadInst *LI = dyn_cast(DI->getOperand(0)); @@ -476,7 +519,7 @@ bool InterleavedAccess::lowerDeinterleaveIntrinsic( return true; } -bool InterleavedAccess::lowerInterleaveIntrinsic( +bool InterleavedAccessImpl::lowerInterleaveIntrinsic( IntrinsicInst *II, SmallVector &DeadInsts) { if (!II->hasOneUse()) return false; @@ -498,18 +541,7 @@ bool InterleavedAccess::lowerInterleaveIntrinsic( return true; } -bool InterleavedAccess::runOnFunction(Function &F) { - auto *TPC = getAnalysisIfAvailable(); - if (!TPC || !LowerInterleavedAccesses) - return false; - - LLVM_DEBUG(dbgs() << "*** " << getPassName() << ": " << F.getName() << "\n"); - - DT = &getAnalysis().getDomTree(); - auto &TM = TPC->getTM(); - TLI = TM.getSubtargetImpl(F)->getTargetLowering(); - MaxFactor = TLI->getMaxSupportedInterleaveFactor(); - +bool InterleavedAccessImpl::runOnFunction(Function &F) { // Holds dead instructions that will be erased later. SmallVector DeadInsts; bool Changed = false; diff --git a/llvm/lib/CodeGen/MachineScheduler.cpp b/llvm/lib/CodeGen/MachineScheduler.cpp index c51ef33bfe54..886137d86f87 100644 --- a/llvm/lib/CodeGen/MachineScheduler.cpp +++ b/llvm/lib/CodeGen/MachineScheduler.cpp @@ -747,9 +747,9 @@ void ScheduleDAGMI::finishBlock() { ScheduleDAGInstrs::finishBlock(); } -/// enterRegion - Called back from MachineScheduler::runOnMachineFunction after -/// crossing a scheduling boundary. [begin, end) includes all instructions in -/// the region, including the boundary itself and single-instruction regions +/// enterRegion - Called back from PostMachineScheduler::runOnMachineFunction +/// after crossing a scheduling boundary. [begin, end) includes all instructions +/// in the region, including the boundary itself and single-instruction regions /// that don't get scheduled. void ScheduleDAGMI::enterRegion(MachineBasicBlock *bb, MachineBasicBlock::iterator begin, @@ -793,9 +793,9 @@ bool ScheduleDAGMI::checkSchedLimit() { } /// Per-region scheduling driver, called back from -/// MachineScheduler::runOnMachineFunction. This is a simplified driver that -/// does not consider liveness or register pressure. It is useful for PostRA -/// scheduling and potentially other custom schedulers. +/// PostMachineScheduler::runOnMachineFunction. This is a simplified driver +/// that does not consider liveness or register pressure. It is useful for +/// PostRA scheduling and potentially other custom schedulers. void ScheduleDAGMI::schedule() { LLVM_DEBUG(dbgs() << "ScheduleDAGMI::schedule starting\n"); LLVM_DEBUG(SchedImpl->dumpPolicy()); diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp index aaf9bd740d13..a015d9bbd2d3 100644 --- a/llvm/lib/CodeGen/MachineVerifier.cpp +++ b/llvm/lib/CodeGen/MachineVerifier.cpp @@ -1812,6 +1812,29 @@ void MachineVerifier::verifyPreISelGenericInstruction(const MachineInstr *MI) { } break; } + case TargetOpcode::G_PREFETCH: { + const MachineOperand &AddrOp = MI->getOperand(0); + if (!AddrOp.isReg() || !MRI->getType(AddrOp.getReg()).isPointer()) { + report("addr operand must be a pointer", &AddrOp, 0); + break; + } + const MachineOperand &RWOp = MI->getOperand(1); + if (!RWOp.isImm() || (uint64_t)RWOp.getImm() >= 2) { + report("rw operand must be an immediate 0-1", &RWOp, 1); + break; + } + const MachineOperand &LocalityOp = MI->getOperand(2); + if (!LocalityOp.isImm() || (uint64_t)LocalityOp.getImm() >= 4) { + report("locality operand must be an immediate 0-3", &LocalityOp, 2); + break; + } + const MachineOperand &CacheTypeOp = MI->getOperand(3); + if (!CacheTypeOp.isImm() || (uint64_t)CacheTypeOp.getImm() >= 2) { + report("cache type operand must be an immediate 0-1", &CacheTypeOp, 3); + break; + } + break; + } case TargetOpcode::G_ASSERT_ALIGN: { if (MI->getOperand(2).getImm() < 1) report("alignment immediate must be >= 1", MI); diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 2a3425a42607..f35f663d6ba1 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -20940,8 +20940,8 @@ SDValue DAGCombiner::replaceStoreOfFPConstant(StoreSDNode *ST) { Ptr, ST->getMemOperand()); } - if (ST->isSimple() && - TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32)) { + if (ST->isSimple() && TLI.isOperationLegalOrCustom(ISD::STORE, MVT::i32) && + !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) { // Many FP stores are not made apparent until after legalize, e.g. for // argument passing. Since this is so common, custom legalize the // 64-bit integer store into two 32-bit stores. @@ -22243,6 +22243,19 @@ SDValue DAGCombiner::visitEXTRACT_VECTOR_ELT(SDNode *N) { unsigned NumElts = VecVT.getVectorNumElements(); unsigned VecEltBitWidth = VecVT.getScalarSizeInBits(); + // See if the extracted element is constant, in which case fold it if its + // a legal fp immediate. + if (IndexC && ScalarVT.isFloatingPoint()) { + APInt EltMask = APInt::getOneBitSet(NumElts, IndexC->getZExtValue()); + KnownBits KnownElt = DAG.computeKnownBits(VecOp, EltMask); + if (KnownElt.isConstant()) { + APFloat CstFP = + APFloat(DAG.EVTToAPFloatSemantics(ScalarVT), KnownElt.getConstant()); + if (TLI.isFPImmLegal(CstFP, ScalarVT)) + return DAG.getConstantFP(CstFP, DL, ScalarVT); + } + } + // TODO: These transforms should not require the 'hasOneUse' restriction, but // there are regressions on multiple targets without it. We can end up with a // mess of scalar and vector code if we reduce only part of the DAG to scalar. diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp index 7a54141fa711..5e1f9fbcdde0 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp @@ -461,7 +461,8 @@ SDValue SelectionDAGLegalize::OptimizeFloatStore(StoreSDNode* ST) { ST->getOriginalAlign(), MMOFlags, AAInfo); } - if (CFP->getValueType(0) == MVT::f64) { + if (CFP->getValueType(0) == MVT::f64 && + !TLI.isFPImmLegal(CFP->getValueAPF(), MVT::f64)) { // If this target supports 64-bit registers, do a single 64-bit store. if (TLI.isTypeLegal(MVT::i64)) { SDValue Con = DAG.getConstant(CFP->getValueAPF().bitcastToAPInt(). diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index 630aa4a07d7b..c4605a6b9598 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -2774,6 +2774,8 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) { case ISD::FPOWI: case ISD::FLDEXP: R = SoftPromoteHalfRes_ExpOp(N); break; + case ISD::FFREXP: R = SoftPromoteHalfRes_FFREXP(N); break; + case ISD::LOAD: R = SoftPromoteHalfRes_LOAD(N); break; case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break; case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break; @@ -2902,6 +2904,24 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ExpOp(SDNode *N) { return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); } +SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FFREXP(SDNode *N) { + EVT OVT = N->getValueType(0); + EVT NVT = TLI.getTypeToTransformTo(*DAG.getContext(), OVT); + SDValue Op = GetSoftPromotedHalf(N->getOperand(0)); + SDLoc dl(N); + + // Promote to the larger FP type. + Op = DAG.getNode(GetPromotionOpcode(OVT, NVT), dl, NVT, Op); + + SDValue Res = DAG.getNode(N->getOpcode(), dl, + DAG.getVTList(NVT, N->getValueType(1)), Op); + + ReplaceValueWith(SDValue(N, 1), Res.getValue(1)); + + // Convert back to FP16 as an integer. + return DAG.getNode(GetPromotionOpcode(NVT, OVT), dl, MVT::i16, Res); +} + SDValue DAGTypeLegalizer::SoftPromoteHalfRes_FP_ROUND(SDNode *N) { EVT RVT = N->getValueType(0); EVT SVT = N->getOperand(0).getValueType(); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index e9bd54089d06..9d5931b44ac6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -738,6 +738,7 @@ private: SDValue SoftPromoteHalfRes_FCOPYSIGN(SDNode *N); SDValue SoftPromoteHalfRes_FMAD(SDNode *N); SDValue SoftPromoteHalfRes_ExpOp(SDNode *N); + SDValue SoftPromoteHalfRes_FFREXP(SDNode *N); SDValue SoftPromoteHalfRes_FP_ROUND(SDNode *N); SDValue SoftPromoteHalfRes_LOAD(SDNode *N); SDValue SoftPromoteHalfRes_SELECT(SDNode *N); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 51ae8b703e50..5be1892a44f6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -2881,6 +2881,12 @@ bool SelectionDAG::isSplatValue(SDValue V, const APInt &DemandedElts, } } + // Fallback - this is a splat if all demanded elts are the same constant. + if (computeKnownBits(V, DemandedElts, Depth).isConstant()) { + UndefElts = ~DemandedElts; + return true; + } + return false; } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index ed1c96a87374..4fd76d012a16 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -7123,7 +7123,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, EVT::getIntegerVT(*Context, 8), MachinePointerInfo(I.getArgOperand(0)), /* align */ std::nullopt, Flags); - // Chain the prefetch in parallell with any pending loads, to stay out of + // Chain the prefetch in parallel with any pending loads, to stay out of // the way of later optimizations. PendingLoads.push_back(Result); Result = getRoot(); diff --git a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp index 143a4951c136..9827bd3ff4f1 100644 --- a/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp +++ b/llvm/lib/CodeGen/TargetLoweringObjectFileImpl.cpp @@ -1048,7 +1048,7 @@ MCSection *TargetLoweringObjectFileELF::getSectionForMachineBasicBlock( } else { Name += FunctionSectionName; if (TM.getUniqueBasicBlockSectionNames()) { - if (!Name.endswith(".")) + if (!Name.ends_with(".")) Name += "."; Name += MBB.getSymbol()->getName(); } else { @@ -2310,8 +2310,10 @@ bool TargetLoweringObjectFileXCOFF::ShouldSetSSPCanaryBitInTB( MCSymbol * TargetLoweringObjectFileXCOFF::getEHInfoTableSymbol(const MachineFunction *MF) { - return MF->getMMI().getContext().getOrCreateSymbol( + MCSymbol *EHInfoSym = MF->getMMI().getContext().getOrCreateSymbol( "__ehinfo." + Twine(MF->getFunctionNumber())); + cast(EHInfoSym)->setEHInfo(); + return EHInfoSym; } MCSymbol * @@ -2644,12 +2646,16 @@ MCSection *TargetLoweringObjectFileXCOFF::getSectionForFunctionDescriptor( MCSection *TargetLoweringObjectFileXCOFF::getSectionForTOCEntry( const MCSymbol *Sym, const TargetMachine &TM) const { // Use TE storage-mapping class when large code model is enabled so that - // the chance of needing -bbigtoc is decreased. + // the chance of needing -bbigtoc is decreased. Also, the toc-entry for + // EH info is never referenced directly using instructions so it can be + // allocated with TE storage-mapping class. return getContext().getXCOFFSection( cast(Sym)->getSymbolTableName(), SectionKind::getData(), - XCOFF::CsectProperties( - TM.getCodeModel() == CodeModel::Large ? XCOFF::XMC_TE : XCOFF::XMC_TC, - XCOFF::XTY_SD)); + XCOFF::CsectProperties((TM.getCodeModel() == CodeModel::Large || + cast(Sym)->isEHInfo()) + ? XCOFF::XMC_TE + : XCOFF::XMC_TC, + XCOFF::XTY_SD)); } MCSection *TargetLoweringObjectFileXCOFF::getSectionForLSDA( diff --git a/llvm/lib/CodeGen/WinEHPrepare.cpp b/llvm/lib/CodeGen/WinEHPrepare.cpp index 347bada6f69d..bd14e8104a6a 100644 --- a/llvm/lib/CodeGen/WinEHPrepare.cpp +++ b/llvm/lib/CodeGen/WinEHPrepare.cpp @@ -120,8 +120,8 @@ PreservedAnalyses WinEHPreparePass::run(Function &F, } char WinEHPrepare::ID = 0; -INITIALIZE_PASS(WinEHPrepare, "winehprepare", "Prepare Windows exceptions", - false, false) +INITIALIZE_PASS(WinEHPrepare, DEBUG_TYPE, "Prepare Windows exceptions", false, + false) FunctionPass *llvm::createWinEHPass(bool DemoteCatchSwitchPHIOnly) { return new WinEHPrepare(DemoteCatchSwitchPHIOnly); diff --git a/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp b/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp index 0f01933002c0..66492f7bf804 100644 --- a/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp +++ b/llvm/lib/DebugInfo/DWARF/DWARFDie.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/DebugInfo/DWARF/DWARFDie.h" +#include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/StringRef.h" #include "llvm/BinaryFormat/Dwarf.h" @@ -489,18 +490,23 @@ void DWARFDie::getCallerFrame(uint32_t &CallFile, uint32_t &CallLine, CallDiscriminator = toUnsigned(find(DW_AT_GNU_discriminator), 0); } -std::optional DWARFDie::getTypeSize(uint64_t PointerSize) { - if (auto SizeAttr = find(DW_AT_byte_size)) +static std::optional +getTypeSizeImpl(DWARFDie Die, uint64_t PointerSize, + SmallPtrSetImpl &Visited) { + // Cycle detected? + if (!Visited.insert(Die.getDebugInfoEntry()).second) + return {}; + if (auto SizeAttr = Die.find(DW_AT_byte_size)) if (std::optional Size = SizeAttr->getAsUnsignedConstant()) return Size; - switch (getTag()) { + switch (Die.getTag()) { case DW_TAG_pointer_type: case DW_TAG_reference_type: case DW_TAG_rvalue_reference_type: return PointerSize; case DW_TAG_ptr_to_member_type: { - if (DWARFDie BaseType = getAttributeValueAsReferencedDie(DW_AT_type)) + if (DWARFDie BaseType = Die.getAttributeValueAsReferencedDie(DW_AT_type)) if (BaseType.getTag() == DW_TAG_subroutine_type) return 2 * PointerSize; return PointerSize; @@ -510,19 +516,20 @@ std::optional DWARFDie::getTypeSize(uint64_t PointerSize) { case DW_TAG_volatile_type: case DW_TAG_restrict_type: case DW_TAG_typedef: { - if (DWARFDie BaseType = getAttributeValueAsReferencedDie(DW_AT_type)) - return BaseType.getTypeSize(PointerSize); + if (DWARFDie BaseType = Die.getAttributeValueAsReferencedDie(DW_AT_type)) + return getTypeSizeImpl(BaseType, PointerSize, Visited); break; } case DW_TAG_array_type: { - DWARFDie BaseType = getAttributeValueAsReferencedDie(DW_AT_type); + DWARFDie BaseType = Die.getAttributeValueAsReferencedDie(DW_AT_type); if (!BaseType) return std::nullopt; - std::optional BaseSize = BaseType.getTypeSize(PointerSize); + std::optional BaseSize = + getTypeSizeImpl(BaseType, PointerSize, Visited); if (!BaseSize) return std::nullopt; uint64_t Size = *BaseSize; - for (DWARFDie Child : *this) { + for (DWARFDie Child : Die) { if (Child.getTag() != DW_TAG_subrange_type) continue; @@ -542,13 +549,18 @@ std::optional DWARFDie::getTypeSize(uint64_t PointerSize) { return Size; } default: - if (DWARFDie BaseType = getAttributeValueAsReferencedDie(DW_AT_type)) - return BaseType.getTypeSize(PointerSize); + if (DWARFDie BaseType = Die.getAttributeValueAsReferencedDie(DW_AT_type)) + return getTypeSizeImpl(BaseType, PointerSize, Visited); break; } return std::nullopt; } +std::optional DWARFDie::getTypeSize(uint64_t PointerSize) { + SmallPtrSet Visited; + return getTypeSizeImpl(*this, PointerSize, Visited); +} + /// Helper to dump a DIE with all of its parents, but no siblings. static unsigned dumpParentChain(DWARFDie Die, raw_ostream &OS, unsigned Indent, DIDumpOptions DumpOpts, unsigned Depth = 0) { diff --git a/llvm/lib/DebugInfo/PDB/Native/FormatUtil.cpp b/llvm/lib/DebugInfo/PDB/Native/FormatUtil.cpp index 9c05d585831a..c5999bffc021 100644 --- a/llvm/lib/DebugInfo/PDB/Native/FormatUtil.cpp +++ b/llvm/lib/DebugInfo/PDB/Native/FormatUtil.cpp @@ -9,6 +9,7 @@ #include "llvm/DebugInfo/PDB/Native/FormatUtil.h" #include "llvm/ADT/STLExtras.h" +#include "llvm/ADT/STLForwardCompat.h" #include "llvm/ADT/StringExtras.h" #include "llvm/BinaryFormat/COFF.h" #include "llvm/DebugInfo/CodeView/CodeView.h" @@ -119,9 +120,7 @@ std::string llvm::pdb::formatTypeLeafKind(TypeLeafKind K) { return #EnumName; #include "llvm/DebugInfo/CodeView/CodeViewTypes.def" default: - return formatv("UNKNOWN RECORD ({0:X})", - static_cast>(K)) - .str(); + return formatv("UNKNOWN RECORD ({0:X})", llvm::to_underlying(K)).str(); } } diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt index 23b471481618..5bf23a7ec0bc 100644 --- a/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/CMakeLists.txt @@ -6,6 +6,7 @@ add_llvm_component_library(LLVMOrcDebugging DebugInfoSupport.cpp DebuggerSupport.cpp DebuggerSupportPlugin.cpp + LLJITUtilsCBindings.cpp PerfSupportPlugin.cpp ADDITIONAL_HEADER_DIRS diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp index 9ba6dd90f50d..1668473c0eb4 100644 --- a/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/DebuggerSupport.cpp @@ -39,7 +39,7 @@ Error enableDebuggerSupport(LLJIT &J) { if (!Registrar) return Registrar.takeError(); ObjLinkingLayer->addPlugin(std::make_unique( - ES, std::move(*Registrar), true, true)); + ES, std::move(*Registrar), false, true)); return Error::success(); } case Triple::MachO: { diff --git a/llvm/lib/ExecutionEngine/Orc/Debugging/LLJITUtilsCBindings.cpp b/llvm/lib/ExecutionEngine/Orc/Debugging/LLJITUtilsCBindings.cpp new file mode 100644 index 000000000000..2df5aef733fb --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/Debugging/LLJITUtilsCBindings.cpp @@ -0,0 +1,22 @@ +//===--------- LLJITUtilsCBindings.cpp - Advanced LLJIT features ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm-c/LLJIT.h" +#include "llvm-c/LLJITUtils.h" + +#include "llvm/ExecutionEngine/Orc/Debugging/DebuggerSupport.h" +#include "llvm/ExecutionEngine/Orc/LLJIT.h" + +using namespace llvm; +using namespace llvm::orc; + +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(LLJIT, LLVMOrcLLJITRef) + +LLVMErrorRef LLVMOrcLLJITEnableDebugSupport(LLVMOrcLLJITRef J) { + return wrap(llvm::orc::enableDebuggerSupport(*unwrap(J))); +} diff --git a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp index a0bd9b6266ff..9057300bf043 100644 --- a/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp +++ b/llvm/lib/ExecutionEngine/Orc/MachOPlatform.cpp @@ -15,6 +15,7 @@ #include "llvm/ExecutionEngine/Orc/DebugUtils.h" #include "llvm/ExecutionEngine/Orc/ExecutionUtils.h" #include "llvm/ExecutionEngine/Orc/LookupAndRecordAddrs.h" +#include "llvm/ExecutionEngine/Orc/MachOBuilder.h" #include "llvm/ExecutionEngine/Orc/Shared/ObjectFormats.h" #include "llvm/Support/BinaryByteStream.h" #include "llvm/Support/Debug.h" @@ -89,6 +90,11 @@ public: namespace { +using SPSRegisterSymbolsArgs = + SPSArgList>>; + std::unique_ptr createPlatformGraph(MachOPlatform &MOP, std::string Name) { unsigned PointerSize; @@ -110,122 +116,32 @@ std::unique_ptr createPlatformGraph(MachOPlatform &MOP, jitlink::getGenericEdgeKindName); } -// Generates a MachO header. -class MachOHeaderMaterializationUnit : public MaterializationUnit { -public: - MachOHeaderMaterializationUnit(MachOPlatform &MOP, - const SymbolStringPtr &HeaderStartSymbol) - : MaterializationUnit(createHeaderInterface(MOP, HeaderStartSymbol)), - MOP(MOP) {} - - StringRef getName() const override { return "MachOHeaderMU"; } - - void materialize(std::unique_ptr R) override { - auto G = createPlatformGraph(MOP, ""); - addMachOHeader(*G, MOP, R->getInitializerSymbol()); - MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); - } - - void discard(const JITDylib &JD, const SymbolStringPtr &Sym) override {} - - static void addMachOHeader(jitlink::LinkGraph &G, MachOPlatform &MOP, - const SymbolStringPtr &InitializerSymbol) { - auto &HeaderSection = G.createSection("__header", MemProt::Read); - auto &HeaderBlock = createHeaderBlock(G, HeaderSection); - - // Init symbol is header-start symbol. - G.addDefinedSymbol(HeaderBlock, 0, *InitializerSymbol, - HeaderBlock.getSize(), jitlink::Linkage::Strong, - jitlink::Scope::Default, false, true); - for (auto &HS : AdditionalHeaderSymbols) - G.addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, HeaderBlock.getSize(), - jitlink::Linkage::Strong, jitlink::Scope::Default, - false, true); - } - -private: - struct HeaderSymbol { - const char *Name; - uint64_t Offset; - }; - - static constexpr HeaderSymbol AdditionalHeaderSymbols[] = { - {"___mh_executable_header", 0}}; - - static jitlink::Block &createHeaderBlock(jitlink::LinkGraph &G, - jitlink::Section &HeaderSection) { - MachO::mach_header_64 Hdr; - Hdr.magic = MachO::MH_MAGIC_64; - switch (G.getTargetTriple().getArch()) { - case Triple::aarch64: - Hdr.cputype = MachO::CPU_TYPE_ARM64; - Hdr.cpusubtype = MachO::CPU_SUBTYPE_ARM64_ALL; - break; - case Triple::x86_64: - Hdr.cputype = MachO::CPU_TYPE_X86_64; - Hdr.cpusubtype = MachO::CPU_SUBTYPE_X86_64_ALL; - break; - default: - llvm_unreachable("Unrecognized architecture"); - } - Hdr.filetype = MachO::MH_DYLIB; // Custom file type? - Hdr.ncmds = 0; - Hdr.sizeofcmds = 0; - Hdr.flags = 0; - Hdr.reserved = 0; - - if (G.getEndianness() != llvm::endianness::native) - MachO::swapStruct(Hdr); - - auto HeaderContent = G.allocateContent( - ArrayRef(reinterpret_cast(&Hdr), sizeof(Hdr))); - - return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, - 0); - } - - static MaterializationUnit::Interface - createHeaderInterface(MachOPlatform &MOP, - const SymbolStringPtr &HeaderStartSymbol) { - SymbolFlagsMap HeaderSymbolFlags; - - HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; - for (auto &HS : AdditionalHeaderSymbols) - HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] = - JITSymbolFlags::Exported; - - return MaterializationUnit::Interface(std::move(HeaderSymbolFlags), - HeaderStartSymbol); - } - - MachOPlatform &MOP; -}; - -constexpr MachOHeaderMaterializationUnit::HeaderSymbol - MachOHeaderMaterializationUnit::AdditionalHeaderSymbols[]; - // Creates a Bootstrap-Complete LinkGraph to run deferred actions. class MachOPlatformCompleteBootstrapMaterializationUnit : public MaterializationUnit { public: + using SymbolTableVector = + SmallVector>; + MachOPlatformCompleteBootstrapMaterializationUnit( MachOPlatform &MOP, StringRef PlatformJDName, - SymbolStringPtr CompleteBootstrapSymbol, shared::AllocActions DeferredAAs, + SymbolStringPtr CompleteBootstrapSymbol, SymbolTableVector SymTab, + shared::AllocActions DeferredAAs, ExecutorAddr MachOHeaderAddr, ExecutorAddr PlatformBootstrap, ExecutorAddr PlatformShutdown, ExecutorAddr RegisterJITDylib, ExecutorAddr DeregisterJITDylib, ExecutorAddr RegisterObjectSymbolTable, - ExecutorAddr DeregisterObjectSymbolTable, ExecutorAddr MachOHeaderAddr) + ExecutorAddr DeregisterObjectSymbolTable) : MaterializationUnit( {{{CompleteBootstrapSymbol, JITSymbolFlags::None}}, nullptr}), MOP(MOP), PlatformJDName(PlatformJDName), CompleteBootstrapSymbol(std::move(CompleteBootstrapSymbol)), - DeferredAAs(std::move(DeferredAAs)), - PlatformBootstrap(PlatformBootstrap), + SymTab(std::move(SymTab)), DeferredAAs(std::move(DeferredAAs)), + MachOHeaderAddr(MachOHeaderAddr), PlatformBootstrap(PlatformBootstrap), PlatformShutdown(PlatformShutdown), RegisterJITDylib(RegisterJITDylib), DeregisterJITDylib(DeregisterJITDylib), RegisterObjectSymbolTable(RegisterObjectSymbolTable), - DeregisterObjectSymbolTable(DeregisterObjectSymbolTable), - MachOHeaderAddr(MachOHeaderAddr) {} + DeregisterObjectSymbolTable(DeregisterObjectSymbolTable) {} StringRef getName() const override { return "MachOPlatformCompleteBootstrap"; @@ -242,7 +158,7 @@ public: Linkage::Strong, Scope::Hidden, false, true); // Reserve space for the stolen actions, plus two extras. - G->allocActions().reserve(DeferredAAs.size() + 2); + G->allocActions().reserve(DeferredAAs.size() + 3); // 1. Bootstrap the platform support code. G->allocActions().push_back( @@ -258,7 +174,14 @@ public: cantFail(WrapperFunctionCall::Create>( DeregisterJITDylib, MachOHeaderAddr))}); - // 3. Add the deferred actions to the graph. + // 3. Register deferred symbols. + G->allocActions().push_back( + {cantFail(WrapperFunctionCall::Create( + RegisterObjectSymbolTable, MachOHeaderAddr, SymTab)), + cantFail(WrapperFunctionCall::Create( + DeregisterObjectSymbolTable, MachOHeaderAddr, SymTab))}); + + // 4. Add the deferred actions to the graph. std::move(DeferredAAs.begin(), DeferredAAs.end(), std::back_inserter(G->allocActions())); @@ -271,14 +194,15 @@ private: MachOPlatform &MOP; StringRef PlatformJDName; SymbolStringPtr CompleteBootstrapSymbol; + SymbolTableVector SymTab; shared::AllocActions DeferredAAs; + ExecutorAddr MachOHeaderAddr; ExecutorAddr PlatformBootstrap; ExecutorAddr PlatformShutdown; ExecutorAddr RegisterJITDylib; ExecutorAddr DeregisterJITDylib; ExecutorAddr RegisterObjectSymbolTable; ExecutorAddr DeregisterObjectSymbolTable; - ExecutorAddr MachOHeaderAddr; }; static StringRef ObjCRuntimeObjectSectionsData[] = { @@ -335,6 +259,7 @@ Expected> MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, std::unique_ptr OrcRuntime, + MachOHeaderMUBuilder BuildMachOHeaderMU, std::optional RuntimeAliases) { // If the target is not supported then bail out immediately. @@ -365,8 +290,9 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, // Create the instance. Error Err = Error::success(); - auto P = std::unique_ptr(new MachOPlatform( - ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), Err)); + auto P = std::unique_ptr( + new MachOPlatform(ES, ObjLinkingLayer, PlatformJD, std::move(OrcRuntime), + std::move(BuildMachOHeaderMU), Err)); if (Err) return std::move(Err); return std::move(P); @@ -375,6 +301,7 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, Expected> MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, const char *OrcRuntimePath, + MachOHeaderMUBuilder BuildMachOHeaderMU, std::optional RuntimeAliases) { // Create a generator for the ORC runtime archive. @@ -385,12 +312,11 @@ MachOPlatform::Create(ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, return Create(ES, ObjLinkingLayer, PlatformJD, std::move(*OrcRuntimeArchiveGenerator), - std::move(RuntimeAliases)); + std::move(BuildMachOHeaderMU), std::move(RuntimeAliases)); } Error MachOPlatform::setupJITDylib(JITDylib &JD) { - if (auto Err = JD.define(std::make_unique( - *this, MachOHeaderStartSymbol))) + if (auto Err = JD.define(BuildMachOHeaderMU(*this))) return Err; return ES.lookup({&JD}, MachOHeaderStartSymbol).takeError(); @@ -505,8 +431,10 @@ MachOPlatform::flagsForSymbol(jitlink::Symbol &Sym) { MachOPlatform::MachOPlatform( ExecutionSession &ES, ObjectLinkingLayer &ObjLinkingLayer, JITDylib &PlatformJD, - std::unique_ptr OrcRuntimeGenerator, Error &Err) - : ES(ES), PlatformJD(PlatformJD), ObjLinkingLayer(ObjLinkingLayer) { + std::unique_ptr OrcRuntimeGenerator, + MachOHeaderMUBuilder BuildMachOHeaderMU, Error &Err) + : ES(ES), PlatformJD(PlatformJD), ObjLinkingLayer(ObjLinkingLayer), + BuildMachOHeaderMU(std::move(BuildMachOHeaderMU)) { ErrorAsOutParameter _(&Err); ObjLinkingLayer.addPlugin(std::make_unique(*this)); PlatformJD.addGenerator(std::move(OrcRuntimeGenerator)); @@ -569,8 +497,7 @@ MachOPlatform::MachOPlatform( // the support methods callable. The bootstrap is now complete. // Step (1) Add header materialization unit and request. - if ((Err = PlatformJD.define(std::make_unique( - *this, MachOHeaderStartSymbol)))) + if ((Err = PlatformJD.define(this->BuildMachOHeaderMU(*this)))) return; if ((Err = ES.lookup(&PlatformJD, MachOHeaderStartSymbol).takeError())) return; @@ -601,10 +528,11 @@ MachOPlatform::MachOPlatform( if ((Err = PlatformJD.define( std::make_unique( *this, PlatformJD.getName(), BootstrapCompleteSymbol, - std::move(BI.DeferredAAs), PlatformBootstrap.Addr, + std::move(BI.SymTab), std::move(BI.DeferredAAs), + BI.MachOHeaderAddr, PlatformBootstrap.Addr, PlatformShutdown.Addr, RegisterJITDylib.Addr, DeregisterJITDylib.Addr, RegisterObjectSymbolTable.Addr, - DeregisterObjectSymbolTable.Addr, BI.MachOHeaderAddr)))) + DeregisterObjectSymbolTable.Addr)))) return; if ((Err = ES.lookup(makeJITDylibSearchOrder( &PlatformJD, JITDylibLookupFlags::MatchAllSymbols), @@ -1714,16 +1642,17 @@ Error MachOPlatform::MachOPlatformPlugin::addSymbolTableRegistration( HeaderAddr = I->second; } - SmallVector> - SymTab; + SymbolTableVector LocalSymTab; + auto &SymTab = LLVM_LIKELY(!InBootstrapPhase) ? LocalSymTab + : MP.Bootstrap.load()->SymTab; for (auto &[OriginalSymbol, NameSym] : JITSymTabInfo) SymTab.push_back({NameSym->getAddress(), OriginalSymbol->getAddress(), flagsForSymbol(*OriginalSymbol)}); - using SPSRegisterSymbolsArgs = - SPSArgList>>; + // Bail out if we're in the bootstrap phase -- registration of thees symbols + // will be attached to the bootstrap graph. + if (LLVM_UNLIKELY(InBootstrapPhase)) + return Error::success(); shared::AllocActions &allocActions = LLVM_LIKELY(!InBootstrapPhase) ? G.allocActions() @@ -1736,5 +1665,97 @@ Error MachOPlatform::MachOPlatformPlugin::addSymbolTableRegistration( return Error::success(); } + +template +jitlink::Block &createTrivialHeaderBlock(MachOPlatform &MOP, + jitlink::LinkGraph &G, + jitlink::Section &HeaderSection) { + auto HdrInfo = + getMachOHeaderInfoFromTriple(MOP.getExecutionSession().getTargetTriple()); + MachOBuilder B(HdrInfo.PageSize); + + B.Header.filetype = MachO::MH_DYLIB; + B.Header.cputype = HdrInfo.CPUType; + B.Header.cpusubtype = HdrInfo.CPUSubType; + + auto HeaderContent = G.allocateBuffer(B.layout()); + B.write(HeaderContent); + + return G.createContentBlock(HeaderSection, HeaderContent, ExecutorAddr(), 8, + 0); +} + +SimpleMachOHeaderMU::SimpleMachOHeaderMU(MachOPlatform &MOP, + SymbolStringPtr HeaderStartSymbol) + : MaterializationUnit( + createHeaderInterface(MOP, std::move(HeaderStartSymbol))), + MOP(MOP) {} + +void SimpleMachOHeaderMU::materialize( + std::unique_ptr R) { + auto G = createPlatformGraph(MOP, ""); + addMachOHeader(R->getTargetJITDylib(), *G, R->getInitializerSymbol()); + MOP.getObjectLinkingLayer().emit(std::move(R), std::move(G)); +} + +void SimpleMachOHeaderMU::discard(const JITDylib &JD, + const SymbolStringPtr &Sym) {} + +void SimpleMachOHeaderMU::addMachOHeader( + JITDylib &JD, jitlink::LinkGraph &G, + const SymbolStringPtr &InitializerSymbol) { + auto &HeaderSection = G.createSection("__header", MemProt::Read); + auto &HeaderBlock = createHeaderBlock(JD, G, HeaderSection); + + // Init symbol is header-start symbol. + G.addDefinedSymbol(HeaderBlock, 0, *InitializerSymbol, HeaderBlock.getSize(), + jitlink::Linkage::Strong, jitlink::Scope::Default, false, + true); + for (auto &HS : AdditionalHeaderSymbols) + G.addDefinedSymbol(HeaderBlock, HS.Offset, HS.Name, HeaderBlock.getSize(), + jitlink::Linkage::Strong, jitlink::Scope::Default, false, + true); +} + +jitlink::Block & +SimpleMachOHeaderMU::createHeaderBlock(JITDylib &JD, jitlink::LinkGraph &G, + jitlink::Section &HeaderSection) { + switch (MOP.getExecutionSession().getTargetTriple().getArch()) { + case Triple::aarch64: + case Triple::x86_64: + return createTrivialHeaderBlock(MOP, G, HeaderSection); + default: + llvm_unreachable("Unsupported architecture"); + } +} + +MaterializationUnit::Interface SimpleMachOHeaderMU::createHeaderInterface( + MachOPlatform &MOP, const SymbolStringPtr &HeaderStartSymbol) { + SymbolFlagsMap HeaderSymbolFlags; + + HeaderSymbolFlags[HeaderStartSymbol] = JITSymbolFlags::Exported; + for (auto &HS : AdditionalHeaderSymbols) + HeaderSymbolFlags[MOP.getExecutionSession().intern(HS.Name)] = + JITSymbolFlags::Exported; + + return MaterializationUnit::Interface(std::move(HeaderSymbolFlags), + HeaderStartSymbol); +} + +MachOHeaderInfo getMachOHeaderInfoFromTriple(const Triple &TT) { + switch (TT.getArch()) { + case Triple::aarch64: + return {/* PageSize = */ 16 * 1024, + /* CPUType = */ MachO::CPU_TYPE_ARM64, + /* CPUSubType = */ MachO::CPU_SUBTYPE_ARM64_ALL}; + case Triple::x86_64: + return {/* PageSize = */ 4 * 1024, + /* CPUType = */ MachO::CPU_TYPE_X86_64, + /* CPUSubType = */ MachO::CPU_SUBTYPE_X86_64_ALL}; + default: + llvm_unreachable("Unrecognized architecture"); + } +} + } // End namespace orc. } // End namespace llvm. diff --git a/llvm/lib/Frontend/HLSL/HLSLResource.cpp b/llvm/lib/Frontend/HLSL/HLSLResource.cpp index 59f730d8a495..709fe3212623 100644 --- a/llvm/lib/Frontend/HLSL/HLSLResource.cpp +++ b/llvm/lib/Frontend/HLSL/HLSLResource.cpp @@ -27,30 +27,37 @@ StringRef FrontendResource::getSourceType() { return cast(Entry->getOperand(1))->getString(); } -uint32_t FrontendResource::FrontendResource::getResourceKind() { +ResourceKind FrontendResource::getResourceKind() { + return static_cast( + cast( + cast(Entry->getOperand(2))->getValue()) + ->getLimitedValue()); +} +bool FrontendResource::getIsROV() { return cast( - cast(Entry->getOperand(2))->getValue()) + cast(Entry->getOperand(3))->getValue()) ->getLimitedValue(); } uint32_t FrontendResource::getResourceIndex() { return cast( - cast(Entry->getOperand(3))->getValue()) + cast(Entry->getOperand(4))->getValue()) ->getLimitedValue(); } uint32_t FrontendResource::getSpace() { return cast( - cast(Entry->getOperand(4))->getValue()) + cast(Entry->getOperand(5))->getValue()) ->getLimitedValue(); } FrontendResource::FrontendResource(GlobalVariable *GV, StringRef TypeStr, - ResourceKind RK, uint32_t ResIndex, - uint32_t Space) { + ResourceKind RK, bool IsROV, + uint32_t ResIndex, uint32_t Space) { auto &Ctx = GV->getContext(); IRBuilder<> B(Ctx); Entry = MDNode::get( Ctx, {ValueAsMetadata::get(GV), MDString::get(Ctx, TypeStr), ConstantAsMetadata::get(B.getInt32(static_cast(RK))), + ConstantAsMetadata::get(B.getInt1(IsROV)), ConstantAsMetadata::get(B.getInt32(ResIndex)), ConstantAsMetadata::get(B.getInt32(Space))}); } diff --git a/llvm/lib/Frontend/Offloading/Utility.cpp b/llvm/lib/Frontend/Offloading/Utility.cpp index 1c08f02c17f5..25f609517ebe 100644 --- a/llvm/lib/Frontend/Offloading/Utility.cpp +++ b/llvm/lib/Frontend/Offloading/Utility.cpp @@ -29,7 +29,7 @@ StructType *offloading::getEntryTy(Module &M) { // TODO: Rework this interface to be more generic. void offloading::emitOffloadingEntry(Module &M, Constant *Addr, StringRef Name, - uint64_t Size, int32_t Flags, + uint64_t Size, int32_t Flags, int32_t Data, StringRef SectionName) { llvm::Triple Triple(M.getTargetTriple()); @@ -51,7 +51,7 @@ void offloading::emitOffloadingEntry(Module &M, Constant *Addr, StringRef Name, ConstantExpr::getPointerBitCastOrAddrSpaceCast(Str, Int8PtrTy), ConstantInt::get(SizeTy, Size), ConstantInt::get(Int32Ty, Flags), - ConstantInt::get(Int32Ty, 0), + ConstantInt::get(Int32Ty, Data), }; Constant *EntryInitializer = ConstantStruct::get(getEntryTy(M), EntryData); diff --git a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp index 4c1def507df4..be9a44885f66 100644 --- a/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp +++ b/llvm/lib/Frontend/OpenMP/OMPIRBuilder.cpp @@ -6393,7 +6393,7 @@ void OpenMPIRBuilder::createOffloadEntry(Constant *ID, Constant *Addr, StringRef Name) { if (!Config.isGPU()) { llvm::offloading::emitOffloadingEntry( - M, ID, Name.empty() ? Addr->getName() : Name, Size, Flags, + M, ID, Name.empty() ? Addr->getName() : Name, Size, Flags, /*Data=*/0, "omp_offloading_entries"); return; } diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp index bff64e3a15a2..95cdec722062 100644 --- a/llvm/lib/IR/AsmWriter.cpp +++ b/llvm/lib/IR/AsmWriter.cpp @@ -3520,15 +3520,15 @@ static void printMetadataIdentifier(StringRef Name, if (Name.empty()) { Out << " "; } else { - if (isalpha(static_cast(Name[0])) || Name[0] == '-' || - Name[0] == '$' || Name[0] == '.' || Name[0] == '_') - Out << Name[0]; + unsigned char FirstC = static_cast(Name[0]); + if (isalpha(FirstC) || FirstC == '-' || FirstC == '$' || FirstC == '.' || + FirstC == '_') + Out << FirstC; else - Out << '\\' << hexdigit(Name[0] >> 4) << hexdigit(Name[0] & 0x0F); + Out << '\\' << hexdigit(FirstC >> 4) << hexdigit(FirstC & 0x0F); for (unsigned i = 1, e = Name.size(); i != e; ++i) { unsigned char C = Name[i]; - if (isalnum(static_cast(C)) || C == '-' || C == '$' || - C == '.' || C == '_') + if (isalnum(C) || C == '-' || C == '$' || C == '.' || C == '_') Out << C; else Out << '\\' << hexdigit(C >> 4) << hexdigit(C & 0x0F); diff --git a/llvm/lib/IR/AutoUpgrade.cpp b/llvm/lib/IR/AutoUpgrade.cpp index 67ee7b7d97e9..645691f44179 100644 --- a/llvm/lib/IR/AutoUpgrade.cpp +++ b/llvm/lib/IR/AutoUpgrade.cpp @@ -1293,7 +1293,8 @@ static bool UpgradeIntrinsicFunction1(Function *F, Function *&NewFn) { } auto *ST = dyn_cast(F->getReturnType()); - if (ST && (!ST->isLiteral() || ST->isPacked())) { + if (ST && (!ST->isLiteral() || ST->isPacked()) && + F->getIntrinsicID() != Intrinsic::not_intrinsic) { // Replace return type with literal non-packed struct. Only do this for // intrinsics declared to return a struct, not for intrinsics with // overloaded return type, in which case the exact struct type will be diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp index 7832028bf367..96629de8a753 100644 --- a/llvm/lib/IR/Core.cpp +++ b/llvm/lib/IR/Core.cpp @@ -43,6 +43,8 @@ using namespace llvm; +DEFINE_SIMPLE_CONVERSION_FUNCTIONS(OperandBundleDef, LLVMOperandBundleRef) + #define DEBUG_TYPE "ir" void llvm::initializeCore(PassRegistry &Registry) { @@ -2567,6 +2569,34 @@ void LLVMRemoveGlobalIFunc(LLVMValueRef IFunc) { unwrap(IFunc)->removeFromParent(); } +/*--.. Operations on operand bundles........................................--*/ + +LLVMOperandBundleRef LLVMCreateOperandBundle(const char *Tag, size_t TagLen, + LLVMValueRef *Args, + unsigned NumArgs) { + return wrap(new OperandBundleDef(std::string(Tag, TagLen), + ArrayRef(unwrap(Args), NumArgs))); +} + +void LLVMDisposeOperandBundle(LLVMOperandBundleRef Bundle) { + delete unwrap(Bundle); +} + +const char *LLVMGetOperandBundleTag(LLVMOperandBundleRef Bundle, size_t *Len) { + StringRef Str = unwrap(Bundle)->getTag(); + *Len = Str.size(); + return Str.data(); +} + +unsigned LLVMGetNumOperandBundleArgs(LLVMOperandBundleRef Bundle) { + return unwrap(Bundle)->inputs().size(); +} + +LLVMValueRef LLVMGetOperandBundleArgAtIndex(LLVMOperandBundleRef Bundle, + unsigned Index) { + return wrap(unwrap(Bundle)->inputs()[Index]); +} + /*--.. Operations on basic blocks ..........................................--*/ LLVMValueRef LLVMBasicBlockAsValue(LLVMBasicBlockRef BB) { @@ -2858,6 +2888,16 @@ LLVMTypeRef LLVMGetCalledFunctionType(LLVMValueRef Instr) { return wrap(unwrap(Instr)->getFunctionType()); } +unsigned LLVMGetNumOperandBundles(LLVMValueRef C) { + return unwrap(C)->getNumOperandBundles(); +} + +LLVMOperandBundleRef LLVMGetOperandBundleAtIndex(LLVMValueRef C, + unsigned Index) { + return wrap( + new OperandBundleDef(unwrap(C)->getOperandBundleAt(Index))); +} + /*--.. Operations on call instructions (only) ..............................--*/ LLVMBool LLVMIsTailCall(LLVMValueRef Call) { @@ -3140,6 +3180,20 @@ LLVMValueRef LLVMBuildInvoke2(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, ArrayRef(unwrap(Args), NumArgs), Name)); } +LLVMValueRef LLVMBuildInvokeWithOperandBundles( + LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, LLVMValueRef *Args, + unsigned NumArgs, LLVMBasicBlockRef Then, LLVMBasicBlockRef Catch, + LLVMOperandBundleRef *Bundles, unsigned NumBundles, const char *Name) { + SmallVector OBs; + for (auto *Bundle : ArrayRef(Bundles, NumBundles)) { + OperandBundleDef *OB = unwrap(Bundle); + OBs.push_back(*OB); + } + return wrap(unwrap(B)->CreateInvoke( + unwrap(Ty), unwrap(Fn), unwrap(Then), unwrap(Catch), + ArrayRef(unwrap(Args), NumArgs), OBs, Name)); +} + LLVMValueRef LLVMBuildLandingPad(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef PersFn, unsigned NumClauses, const char *Name) { @@ -3878,6 +3932,21 @@ LLVMValueRef LLVMBuildCall2(LLVMBuilderRef B, LLVMTypeRef Ty, LLVMValueRef Fn, ArrayRef(unwrap(Args), NumArgs), Name)); } +LLVMValueRef +LLVMBuildCallWithOperandBundles(LLVMBuilderRef B, LLVMTypeRef Ty, + LLVMValueRef Fn, LLVMValueRef *Args, + unsigned NumArgs, LLVMOperandBundleRef *Bundles, + unsigned NumBundles, const char *Name) { + FunctionType *FTy = unwrap(Ty); + SmallVector OBs; + for (auto *Bundle : ArrayRef(Bundles, NumBundles)) { + OperandBundleDef *OB = unwrap(Bundle); + OBs.push_back(*OB); + } + return wrap(unwrap(B)->CreateCall( + FTy, unwrap(Fn), ArrayRef(unwrap(Args), NumArgs), OBs, Name)); +} + LLVMValueRef LLVMBuildSelect(LLVMBuilderRef B, LLVMValueRef If, LLVMValueRef Then, LLVMValueRef Else, const char *Name) { diff --git a/llvm/lib/IR/DebugProgramInstruction.cpp b/llvm/lib/IR/DebugProgramInstruction.cpp index 6a4ee9d61010..df45c6ea3a77 100644 --- a/llvm/lib/IR/DebugProgramInstruction.cpp +++ b/llvm/lib/IR/DebugProgramInstruction.cpp @@ -35,10 +35,9 @@ DPValue::DPValue(const DPValue &DPV) DbgLoc(DPV.getDebugLoc()), Type(DPV.getType()) {} DPValue::DPValue(Metadata *Location, DILocalVariable *DV, DIExpression *Expr, - const DILocation *DI) + const DILocation *DI, LocationType Type) : DebugValueUser(Location), Variable(DV), Expression(Expr), DbgLoc(DI), - Type(LocationType::Value) { -} + Type(Type) {} void DPValue::deleteInstr() { delete this; } diff --git a/llvm/lib/IR/Type.cpp b/llvm/lib/IR/Type.cpp index 3d2e203a20da..a185ca3fb8dc 100644 --- a/llvm/lib/IR/Type.cpp +++ b/llvm/lib/IR/Type.cpp @@ -141,16 +141,9 @@ bool Type::canLosslesslyBitCastTo(Type *Ty) const { Ty->getPrimitiveSizeInBits().getFixedValue() == 8192) return true; - // At this point we have only various mismatches of the first class types - // remaining and ptr->ptr. Just select the lossless conversions. Everything - // else is not lossless. Conservatively assume we can't losslessly convert - // between pointers with different address spaces. - if (auto *PTy = dyn_cast(this)) { - if (auto *OtherPTy = dyn_cast(Ty)) - return PTy->getAddressSpace() == OtherPTy->getAddressSpace(); - return false; - } - return false; // Other types have no identity values + // Conservatively assume we can't losslessly convert between pointers with + // different address spaces. + return false; } bool Type::isEmptyTy() const { diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index f137f0468c3c..c87f164bdd0f 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -5374,7 +5374,7 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) { Check(cast(Call.getArgOperand(1))->getZExtValue() < 2, "rw argument to llvm.prefetch must be 0-1", Call); Check(cast(Call.getArgOperand(2))->getZExtValue() < 4, - "locality argument to llvm.prefetch must be 0-4", Call); + "locality argument to llvm.prefetch must be 0-3", Call); Check(cast(Call.getArgOperand(3))->getZExtValue() < 2, "cache type argument to llvm.prefetch must be 0-1", Call); break; diff --git a/llvm/lib/MC/MCAssembler.cpp b/llvm/lib/MC/MCAssembler.cpp index 901a66f15666..def13044dfcc 100644 --- a/llvm/lib/MC/MCAssembler.cpp +++ b/llvm/lib/MC/MCAssembler.cpp @@ -193,9 +193,9 @@ const MCSymbol *MCAssembler::getAtom(const MCSymbol &S) const { return S.getFragment()->getAtom(); } -bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout, - const MCFixup &Fixup, const MCFragment *DF, - MCValue &Target, uint64_t &Value, +bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout, const MCFixup &Fixup, + const MCFragment *DF, MCValue &Target, + const MCSubtargetInfo *STI, uint64_t &Value, bool &WasForced) const { ++stats::evaluateFixup; @@ -227,7 +227,7 @@ bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout, if (IsTarget) return getBackend().evaluateTargetFixup(*this, Layout, Fixup, DF, Target, - Value, WasForced); + STI, Value, WasForced); unsigned FixupFlags = getBackendPtr()->getFixupKindInfo(Fixup.getKind()).Flags; bool IsPCRel = getBackendPtr()->getFixupKindInfo(Fixup.getKind()).Flags & @@ -282,7 +282,8 @@ bool MCAssembler::evaluateFixup(const MCAsmLayout &Layout, } // Let the backend force a relocation if needed. - if (IsResolved && getBackend().shouldForceRelocation(*this, Fixup, Target)) { + if (IsResolved && + getBackend().shouldForceRelocation(*this, Fixup, Target, STI)) { IsResolved = false; WasForced = true; } @@ -796,13 +797,13 @@ void MCAssembler::writeSectionData(raw_ostream &OS, const MCSection *Sec, std::tuple MCAssembler::handleFixup(const MCAsmLayout &Layout, MCFragment &F, - const MCFixup &Fixup) { + const MCFixup &Fixup, const MCSubtargetInfo *STI) { // Evaluate the fixup. MCValue Target; uint64_t FixedValue; bool WasForced; - bool IsResolved = evaluateFixup(Layout, Fixup, &F, Target, FixedValue, - WasForced); + bool IsResolved = + evaluateFixup(Layout, Fixup, &F, Target, STI, FixedValue, WasForced); if (!IsResolved) { // The fixup was unresolved, we need a relocation. Inform the object // writer of the relocation, and give it an opportunity to adjust the @@ -936,7 +937,7 @@ void MCAssembler::layout(MCAsmLayout &Layout) { bool IsResolved; MCValue Target; std::tie(Target, FixedValue, IsResolved) = - handleFixup(Layout, Frag, Fixup); + handleFixup(Layout, Frag, Fixup, STI); getBackend().applyFixup(*this, Fixup, Target, Contents, FixedValue, IsResolved, STI); } @@ -960,7 +961,8 @@ bool MCAssembler::fixupNeedsRelaxation(const MCFixup &Fixup, MCValue Target; uint64_t Value; bool WasForced; - bool Resolved = evaluateFixup(Layout, Fixup, DF, Target, Value, WasForced); + bool Resolved = evaluateFixup(Layout, Fixup, DF, Target, + DF->getSubtargetInfo(), Value, WasForced); if (Target.getSymA() && Target.getSymA()->getKind() == MCSymbolRefExpr::VK_X86_ABS8 && Fixup.getKind() == FK_Data_1) diff --git a/llvm/lib/MC/MCELFStreamer.cpp b/llvm/lib/MC/MCELFStreamer.cpp index 653ff4e9435a..e541090769e9 100644 --- a/llvm/lib/MC/MCELFStreamer.cpp +++ b/llvm/lib/MC/MCELFStreamer.cpp @@ -501,7 +501,6 @@ void MCELFStreamer::finalizeCGProfileEntry(const MCSymbolRefExpr *&SRE, SRE->getLoc()); } const MCConstantExpr *MCOffset = MCConstantExpr::create(Offset, getContext()); - MCObjectStreamer::visitUsedExpr(*SRE); if (std::optional> Err = MCObjectStreamer::emitRelocDirective( *MCOffset, "BFD_RELOC_NONE", SRE, SRE->getLoc(), diff --git a/llvm/lib/MC/MCObjectStreamer.cpp b/llvm/lib/MC/MCObjectStreamer.cpp index 3cf7b4359cab..d11ccfb5e269 100644 --- a/llvm/lib/MC/MCObjectStreamer.cpp +++ b/llvm/lib/MC/MCObjectStreamer.cpp @@ -797,8 +797,9 @@ MCObjectStreamer::emitRelocDirective(const MCExpr &Offset, StringRef Name, return std::make_pair(true, std::string("unknown relocation name")); MCFixupKind Kind = *MaybeKind; - - if (Expr == nullptr) + if (Expr) + visitUsedExpr(*Expr); + else Expr = MCSymbolRefExpr::create(getContext().createTempSymbol(), getContext()); diff --git a/llvm/lib/MC/MCSchedule.cpp b/llvm/lib/MC/MCSchedule.cpp index 990a693559a7..4f7125864c5a 100644 --- a/llvm/lib/MC/MCSchedule.cpp +++ b/llvm/lib/MC/MCSchedule.cpp @@ -20,8 +20,8 @@ using namespace llvm; -static_assert(std::is_pod::value, - "We shouldn't have a static constructor here"); +static_assert(std::is_trivial_v, + "MCSchedModel is required to be a trivial type"); const MCSchedModel MCSchedModel::Default = {DefaultIssueWidth, DefaultMicroOpBufferSize, DefaultLoopMicroOpBufferSize, @@ -30,7 +30,7 @@ const MCSchedModel MCSchedModel::Default = {DefaultIssueWidth, DefaultMispredictPenalty, false, true, - false /*EnableIntervals*/, + /*EnableIntervals=*/false, 0, nullptr, nullptr, diff --git a/llvm/lib/MC/MCSubtargetInfo.cpp b/llvm/lib/MC/MCSubtargetInfo.cpp index 8ee823e0377b..cf3aba17fc3d 100644 --- a/llvm/lib/MC/MCSubtargetInfo.cpp +++ b/llvm/lib/MC/MCSubtargetInfo.cpp @@ -214,7 +214,7 @@ void MCSubtargetInfo::InitMCProcessorInfo(StringRef CPU, StringRef TuneCPU, if (!TuneCPU.empty()) CPUSchedModel = &getSchedModelForCPU(TuneCPU); else - CPUSchedModel = &MCSchedModel::GetDefaultSchedModel(); + CPUSchedModel = &MCSchedModel::Default; } void MCSubtargetInfo::setDefaultFeatures(StringRef CPU, StringRef TuneCPU, @@ -319,7 +319,7 @@ const MCSchedModel &MCSubtargetInfo::getSchedModelForCPU(StringRef CPU) const { errs() << "'" << CPU << "' is not a recognized processor for this target" << " (ignoring processor)\n"; - return MCSchedModel::GetDefaultSchedModel(); + return MCSchedModel::Default; } assert(CPUEntry->SchedModel && "Missing processor SchedModel value"); return *CPUEntry->SchedModel; diff --git a/llvm/lib/ObjCopy/CommonConfig.cpp b/llvm/lib/ObjCopy/CommonConfig.cpp index e85715d0c44c..f44e70d996b2 100644 --- a/llvm/lib/ObjCopy/CommonConfig.cpp +++ b/llvm/lib/ObjCopy/CommonConfig.cpp @@ -7,6 +7,7 @@ //===----------------------------------------------------------------------===// #include "llvm/ObjCopy/CommonConfig.h" +#include "llvm/Support/Errc.h" namespace llvm { namespace objcopy { @@ -38,6 +39,12 @@ NameOrPattern::create(StringRef Pattern, MatchStyle MS, IsPositiveMatch); } case MatchStyle::Regex: { + Regex RegEx(Pattern); + std::string Err; + if (!RegEx.isValid(Err)) + return createStringError(errc::invalid_argument, + "cannot compile regular expression \'" + + Pattern + "\': " + Err); SmallVector Data; return NameOrPattern(std::make_shared( ("^" + Pattern.ltrim('^').rtrim('$') + "$").toStringRef(Data))); diff --git a/llvm/lib/Object/ELF.cpp b/llvm/lib/Object/ELF.cpp index 1d73a6ffa73f..36847d1a2a42 100644 --- a/llvm/lib/Object/ELF.cpp +++ b/llvm/lib/Object/ELF.cpp @@ -273,6 +273,7 @@ StringRef llvm::object::getELFSectionTypeName(uint32_t Machine, unsigned Type) { break; case ELF::EM_AARCH64: switch (Type) { + STRINGIFY_ENUM_CASE(ELF, SHT_AARCH64_AUTH_RELR); STRINGIFY_ENUM_CASE(ELF, SHT_AARCH64_MEMTAG_GLOBALS_DYNAMIC); STRINGIFY_ENUM_CASE(ELF, SHT_AARCH64_MEMTAG_GLOBALS_STATIC); } diff --git a/llvm/lib/ObjectYAML/ELFYAML.cpp b/llvm/lib/ObjectYAML/ELFYAML.cpp index 1da4ea4e3edc..c47d4eaa309d 100644 --- a/llvm/lib/ObjectYAML/ELFYAML.cpp +++ b/llvm/lib/ObjectYAML/ELFYAML.cpp @@ -719,6 +719,7 @@ void ScalarEnumerationTraits::enumeration( ECase(SHT_MSP430_ATTRIBUTES); break; case ELF::EM_AARCH64: + ECase(SHT_AARCH64_AUTH_RELR); ECase(SHT_AARCH64_MEMTAG_GLOBALS_STATIC); ECase(SHT_AARCH64_MEMTAG_GLOBALS_DYNAMIC); break; diff --git a/llvm/lib/ObjectYAML/XCOFFYAML.cpp b/llvm/lib/ObjectYAML/XCOFFYAML.cpp index 44ef33501b65..398b09c72170 100644 --- a/llvm/lib/ObjectYAML/XCOFFYAML.cpp +++ b/llvm/lib/ObjectYAML/XCOFFYAML.cpp @@ -280,47 +280,60 @@ static void auxSymMapping(IO &IO, XCOFFYAML::SectAuxEntForStat &AuxSym) { IO.mapOptional("NumberOfLineNum", AuxSym.NumberOfLineNum); } +template +static void ResetAuxSym(IO &IO, + std::unique_ptr &AuxSym) { + if (!IO.outputting()) + AuxSym.reset(new AuxEntT); +} + void MappingTraits>::mapping( IO &IO, std::unique_ptr &AuxSym) { - assert(!IO.outputting() && "We don't dump aux symbols currently."); const bool Is64 = static_cast(IO.getContext())->Header.Magic == (llvm::yaml::Hex16)XCOFF::XCOFF64; + XCOFFYAML::AuxSymbolType AuxType; + if (IO.outputting()) + AuxType = AuxSym.get()->Type; IO.mapRequired("Type", AuxType); switch (AuxType) { case XCOFFYAML::AUX_EXCEPT: - if (!Is64) + if (!Is64) { IO.setError("an auxiliary symbol of type AUX_EXCEPT cannot be defined in " "XCOFF32"); - AuxSym.reset(new XCOFFYAML::ExcpetionAuxEnt()); + return; + } + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get())); break; case XCOFFYAML::AUX_FCN: - AuxSym.reset(new XCOFFYAML::FunctionAuxEnt()); + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get()), Is64); break; case XCOFFYAML::AUX_SYM: - AuxSym.reset(new XCOFFYAML::BlockAuxEnt()); + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get()), Is64); break; case XCOFFYAML::AUX_FILE: - AuxSym.reset(new XCOFFYAML::FileAuxEnt()); + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get())); break; case XCOFFYAML::AUX_CSECT: - AuxSym.reset(new XCOFFYAML::CsectAuxEnt()); + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get()), Is64); break; case XCOFFYAML::AUX_SECT: - AuxSym.reset(new XCOFFYAML::SectAuxEntForDWARF()); + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get())); break; case XCOFFYAML::AUX_STAT: - if (Is64) + if (Is64) { IO.setError( "an auxiliary symbol of type AUX_STAT cannot be defined in XCOFF64"); - AuxSym.reset(new XCOFFYAML::SectAuxEntForStat()); + return; + } + ResetAuxSym(IO, AuxSym); auxSymMapping(IO, *cast(AuxSym.get())); break; } @@ -334,8 +347,7 @@ void MappingTraits::mapping(IO &IO, XCOFFYAML::Symbol &S) { IO.mapOptional("Type", S.Type); IO.mapOptional("StorageClass", S.StorageClass); IO.mapOptional("NumberOfAuxEntries", S.NumberOfAuxEntries); - if (!IO.outputting()) - IO.mapOptional("AuxEntries", S.AuxEntries); + IO.mapOptional("AuxEntries", S.AuxEntries); } void MappingTraits::mapping(IO &IO, XCOFFYAML::StringTable &Str) { diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index a5f9b5424358..f26450e94187 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -77,6 +77,7 @@ #include "llvm/CodeGen/ExpandLargeDivRem.h" #include "llvm/CodeGen/ExpandLargeFpConvert.h" #include "llvm/CodeGen/HardwareLoops.h" +#include "llvm/CodeGen/InterleavedAccess.h" #include "llvm/CodeGen/SafeStack.h" #include "llvm/CodeGen/TypePromotion.h" #include "llvm/CodeGen/WasmEHPrepare.h" diff --git a/llvm/lib/Passes/PassBuilderPipelines.cpp b/llvm/lib/Passes/PassBuilderPipelines.cpp index e7f88680655c..5c6c391049a7 100644 --- a/llvm/lib/Passes/PassBuilderPipelines.cpp +++ b/llvm/lib/Passes/PassBuilderPipelines.cpp @@ -810,7 +810,7 @@ void PassBuilder::addPGOInstrPasses(ModulePassManager &MPM, Options.DoCounterPromotion = true; Options.UseBFIInPromotion = IsCS; Options.Atomic = AtomicCounterUpdate; - MPM.addPass(InstrProfiling(Options, IsCS)); + MPM.addPass(InstrProfilingLoweringPass(Options, IsCS)); } void PassBuilder::addPGOInstrPassesForO0( @@ -837,7 +837,7 @@ void PassBuilder::addPGOInstrPassesForO0( Options.DoCounterPromotion = false; Options.UseBFIInPromotion = IsCS; Options.Atomic = AtomicCounterUpdate; - MPM.addPass(InstrProfiling(Options, IsCS)); + MPM.addPass(InstrProfilingLoweringPass(Options, IsCS)); } static InlineParams getInlineParamsFromOptLevel(OptimizationLevel Level) { diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 7462704ec2df..56449906eb65 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -77,7 +77,7 @@ MODULE_PASS("inliner-wrapper-no-mandatory-first", ModuleInlinerWrapperPass(getInlineParams(), false)) MODULE_PASS("insert-gcov-profiling", GCOVProfilerPass()) MODULE_PASS("instrorderfile", InstrOrderFilePass()) -MODULE_PASS("instrprof", InstrProfiling()) +MODULE_PASS("instrprof", InstrProfilingLoweringPass()) MODULE_PASS("internalize", InternalizePass()) MODULE_PASS("invalidate", InvalidateAllAnalysesPass()) MODULE_PASS("iroutliner", IROutlinerPass()) @@ -302,7 +302,7 @@ FUNCTION_PASS("dot-dom-only", DomOnlyPrinter()) FUNCTION_PASS("dot-post-dom", PostDomPrinter()) FUNCTION_PASS("dot-post-dom-only", PostDomOnlyPrinter()) FUNCTION_PASS("dse", DSEPass()) -FUNCTION_PASS("dwarfehprepare", DwarfEHPreparePass(TM)) +FUNCTION_PASS("dwarf-eh-prepare", DwarfEHPreparePass(TM)) FUNCTION_PASS("expand-large-div-rem", ExpandLargeDivRemPass(TM)) FUNCTION_PASS("expand-large-fp-convert", ExpandLargeFpConvertPass(TM)) FUNCTION_PASS("fix-irreducible", FixIrreduciblePass()) @@ -318,6 +318,7 @@ FUNCTION_PASS("inject-tli-mappings", InjectTLIMappings()) FUNCTION_PASS("instcount", InstCountPass()) FUNCTION_PASS("instnamer", InstructionNamerPass()) FUNCTION_PASS("instsimplify", InstSimplifyPass()) +FUNCTION_PASS("interleaved-access", InterleavedAccessPass(TM)) FUNCTION_PASS("invalidate", InvalidateAllAnalysesPass()) FUNCTION_PASS("irce", IRCEPass()) FUNCTION_PASS("jump-threading", JumpThreadingPass()) diff --git a/llvm/lib/Support/CommandLine.cpp b/llvm/lib/Support/CommandLine.cpp index a7e0cae8b855..31f79972125d 100644 --- a/llvm/lib/Support/CommandLine.cpp +++ b/llvm/lib/Support/CommandLine.cpp @@ -2372,7 +2372,7 @@ public: if (Sub == &SubCommand::getTopLevel()) { outs() << "USAGE: " << GlobalParser->ProgramName; - if (Subs.size() > 2) + if (!Subs.empty()) outs() << " [subcommand]"; outs() << " [options]"; } else { diff --git a/llvm/lib/Support/Windows/Path.inc b/llvm/lib/Support/Windows/Path.inc index e4563fd6ed9e..168a63bb2d96 100644 --- a/llvm/lib/Support/Windows/Path.inc +++ b/llvm/lib/Support/Windows/Path.inc @@ -76,7 +76,7 @@ std::error_code widenPath(const Twine &Path8, SmallVectorImpl &Path16, // If the path is a long path, mangled into forward slashes, normalize // back to backslashes here. - if (Path8Str.startswith("//?/")) + if (Path8Str.starts_with("//?/")) llvm::sys::path::native(Path8Str, path::Style::windows_backslash); if (std::error_code EC = UTF8ToUTF16(Path8Str, Path16)) @@ -96,7 +96,7 @@ std::error_code widenPath(const Twine &Path8, SmallVectorImpl &Path16, const char *const LongPathPrefix = "\\\\?\\"; if ((Path16.size() + CurPathLen) < MaxPathLen || - Path8Str.startswith(LongPathPrefix)) + Path8Str.starts_with(LongPathPrefix)) return std::error_code(); if (!IsAbsolute) { diff --git a/llvm/lib/TableGen/JSONBackend.cpp b/llvm/lib/TableGen/JSONBackend.cpp index a9dc30d7f59b..2a3f522a9c0e 100644 --- a/llvm/lib/TableGen/JSONBackend.cpp +++ b/llvm/lib/TableGen/JSONBackend.cpp @@ -11,7 +11,6 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/ArrayRef.h" #include "llvm/Support/Casting.h" #include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" diff --git a/llvm/lib/Target/AArch64/AArch64.td b/llvm/lib/Target/AArch64/AArch64.td index ff256c9a8ccd..c600bcaab2b3 100644 --- a/llvm/lib/Target/AArch64/AArch64.td +++ b/llvm/lib/Target/AArch64/AArch64.td @@ -1480,9 +1480,9 @@ def ProcessorFeatures { FeatureFPARMv8, FeatureFullFP16, FeatureNEON, FeatureRCPC, FeatureSPE, FeatureSSBS, FeaturePerfMon]; - list NeoverseN2 = [HasV8_5aOps, FeatureBF16, FeatureETE, + list NeoverseN2 = [HasV9_0aOps, FeatureBF16, FeatureETE, FeatureMatMulInt8, FeatureMTE, FeatureSVE2, - FeatureSVE2BitPerm, FeatureTRBE, FeatureCrypto, + FeatureSVE2BitPerm, FeatureTRBE, FeaturePerfMon]; list Neoverse512TVB = [HasV8_4aOps, FeatureBF16, FeatureCacheDeepPersist, FeatureCrypto, FeatureFPARMv8, FeatureFP16FML, diff --git a/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp b/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp index 811635faaf1b..ff569e3dce2e 100644 --- a/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp +++ b/llvm/lib/Target/AArch64/AArch64CollectLOH.cpp @@ -101,7 +101,6 @@ #include "AArch64InstrInfo.h" #include "AArch64MachineFunctionInfo.h" #include "llvm/ADT/SmallSet.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Statistic.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunctionPass.h" diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp index 3748f671f123..757471d6a905 100644 --- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp +++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp @@ -38,7 +38,6 @@ #include #include #include -#include #include using namespace llvm; diff --git a/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp b/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp index 32686b25f211..7544786d9f6c 100644 --- a/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp +++ b/llvm/lib/Target/AArch64/AArch64FalkorHWPFFix.cpp @@ -46,7 +46,6 @@ #include "llvm/Support/Debug.h" #include "llvm/Support/DebugCounter.h" #include "llvm/Support/raw_ostream.h" -#include #include #include diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index f36607b03e76..3882e843fb69 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -1651,6 +1651,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM, PredictableSelectIsExpensive = Subtarget->predictableSelectIsExpensive(); IsStrictFPEnabled = true; + setMaxAtomicSizeInBitsSupported(128); } void AArch64TargetLowering::addTypeForNEON(MVT VT) { @@ -13705,11 +13706,17 @@ static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS, assert(VT.getSizeInBits() == SrcVT.getSizeInBits() && "function only supposed to emit natural comparisons"); + APInt SplatValue; + APInt SplatUndef; + unsigned SplatBitSize; + bool HasAnyUndefs; + BuildVectorSDNode *BVN = dyn_cast(RHS.getNode()); - APInt CnstBits(VT.getSizeInBits(), 0); - APInt UndefBits(VT.getSizeInBits(), 0); - bool IsCnst = BVN && resolveBuildVector(BVN, CnstBits, UndefBits); - bool IsZero = IsCnst && (CnstBits == 0); + bool IsCnst = BVN && BVN->isConstantSplat(SplatValue, SplatUndef, + SplatBitSize, HasAnyUndefs); + bool IsZero = IsCnst && SplatValue == 0; + bool IsOne = IsCnst && SplatValue == 1; + bool IsMinusOne = IsCnst && SplatValue.isAllOnes(); if (SrcVT.getVectorElementType().isFloatingPoint()) { switch (CC) { @@ -13778,6 +13785,8 @@ static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS, case AArch64CC::GT: if (IsZero) return DAG.getNode(AArch64ISD::CMGTz, dl, VT, LHS); + if (IsMinusOne) + return DAG.getNode(AArch64ISD::CMGEz, dl, VT, LHS, RHS); return DAG.getNode(AArch64ISD::CMGT, dl, VT, LHS, RHS); case AArch64CC::LE: if (IsZero) @@ -13790,6 +13799,8 @@ static SDValue EmitVectorComparison(SDValue LHS, SDValue RHS, case AArch64CC::LT: if (IsZero) return DAG.getNode(AArch64ISD::CMLTz, dl, VT, LHS); + if (IsOne) + return DAG.getNode(AArch64ISD::CMLEz, dl, VT, LHS); return DAG.getNode(AArch64ISD::CMGT, dl, VT, RHS, LHS); case AArch64CC::HI: return DAG.getNode(AArch64ISD::CMHI, dl, VT, LHS, RHS); @@ -24899,15 +24910,21 @@ AArch64TargetLowering::shouldExpandAtomicLoadInIR(LoadInst *LI) const { : AtomicExpansionKind::LLSC; } -// For the real atomic operations, we have ldxr/stxr up to 128 bits, +// The "default" for integer RMW operations is to expand to an LL/SC loop. +// However, with the LSE instructions (or outline-atomics mode, which provides +// library routines in place of the LSE-instructions), we can directly emit many +// operations instead. +// +// Floating-point operations are always emitted to a cmpxchg loop, because they +// may trigger a trap which aborts an LLSC sequence. TargetLowering::AtomicExpansionKind AArch64TargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *AI) const { + unsigned Size = AI->getType()->getPrimitiveSizeInBits(); + assert(Size <= 128 && "AtomicExpandPass should've handled larger sizes."); + if (AI->isFloatingPointOperation()) return AtomicExpansionKind::CmpXChg; - unsigned Size = AI->getType()->getPrimitiveSizeInBits(); - if (Size > 128) return AtomicExpansionKind::None; - bool CanUseLSE128 = Subtarget->hasLSE128() && Size == 128 && (AI->getOperation() == AtomicRMWInst::Xchg || AI->getOperation() == AtomicRMWInst::Or || diff --git a/llvm/lib/Target/AArch64/AArch64InstrGISel.td b/llvm/lib/Target/AArch64/AArch64InstrGISel.td index 1711360779bf..1c88456560d3 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrGISel.td +++ b/llvm/lib/Target/AArch64/AArch64InstrGISel.td @@ -209,7 +209,7 @@ def G_FCMLTZ : AArch64GenericInstruction { let hasSideEffects = 0; } -def G_PREFETCH : AArch64GenericInstruction { +def G_AARCH64_PREFETCH : AArch64GenericInstruction { let OutOperandList = (outs); let InOperandList = (ins type0:$imm, ptype0:$src1); let hasSideEffects = 1; @@ -287,7 +287,7 @@ def : GINodeEquiv; def : GINodeEquiv; -def : GINodeEquiv; +def : GINodeEquiv; // These are patterns that we only use for GlobalISel via the importer. def : Pat<(f32 (fadd (vector_extract (v2f32 FPR64:$Rn), (i64 0)), diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp index 93b8295f4f3e..50cbd3672fbd 100644 --- a/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.cpp @@ -9532,9 +9532,9 @@ AArch64InstrInfo::probedStackAlloc(MachineBasicBlock::iterator MBBI, .addImm(AArch64_AM::getShifterImm(AArch64_AM::LSL, 0)) .setMIFlags(Flags); - // STR XZR, [SP] - BuildMI(*ExitMBB, ExitMBB->end(), DL, TII->get(AArch64::STRXui)) - .addReg(AArch64::XZR) + // LDR XZR, [SP] + BuildMI(*ExitMBB, ExitMBB->end(), DL, TII->get(AArch64::LDRXui)) + .addReg(AArch64::XZR, RegState::Define) .addReg(AArch64::SP) .addImm(0) .setMIFlags(Flags); diff --git a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp index a4165d0514dc..3687492c3e3e 100644 --- a/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp +++ b/llvm/lib/Target/AArch64/AArch64SLSHardening.cpp @@ -14,7 +14,6 @@ #include "AArch64InstrInfo.h" #include "AArch64Subtarget.h" #include "Utils/AArch64BaseInfo.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/IndirectThunks.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/MachineFunction.h" diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h index fa4c93d5f77a..0b220069a388 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h @@ -291,6 +291,7 @@ public: bool isLegalMaskedGather(Type *DataType, Align Alignment) const { return isLegalMaskedGatherScatter(DataType); } + bool isLegalMaskedScatter(Type *DataType, Align Alignment) const { return isLegalMaskedGatherScatter(DataType); } diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp index 21a412e9360d..a35957c34a59 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp @@ -1127,6 +1127,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) getActionDefinitionsBuilder(G_IS_FPCLASS).lower(); + getActionDefinitionsBuilder(G_PREFETCH).custom(); + getLegacyLegalizerInfo().computeTables(); verify(*ST.getInstrInfo()); } @@ -1176,6 +1178,8 @@ bool AArch64LegalizerInfo::legalizeCustom(LegalizerHelper &Helper, return legalizeExtractVectorElt(MI, MRI, Helper); case TargetOpcode::G_DYN_STACKALLOC: return legalizeDynStackAlloc(MI, Helper); + case TargetOpcode::G_PREFETCH: + return legalizePrefetch(MI, Helper); } llvm_unreachable("expected switch to return"); @@ -1349,30 +1353,6 @@ bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, Value.setReg(ExtValueReg); return true; } - case Intrinsic::prefetch: { - MachineIRBuilder MIB(MI); - auto &AddrVal = MI.getOperand(1); - - int64_t IsWrite = MI.getOperand(2).getImm(); - int64_t Locality = MI.getOperand(3).getImm(); - int64_t IsData = MI.getOperand(4).getImm(); - - bool IsStream = Locality == 0; - if (Locality != 0) { - assert(Locality <= 3 && "Prefetch locality out-of-range"); - // The locality degree is the opposite of the cache speed. - // Put the number the other way around. - // The encoding starts at 0 for level 1 - Locality = 3 - Locality; - } - - unsigned PrfOp = - (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream; - - MIB.buildInstr(AArch64::G_PREFETCH).addImm(PrfOp).add(AddrVal); - MI.eraseFromParent(); - return true; - } case Intrinsic::aarch64_prefetch: { MachineIRBuilder MIB(MI); auto &AddrVal = MI.getOperand(1); @@ -1387,7 +1367,7 @@ bool AArch64LegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, (Target << 1) | // Cache level bits (unsigned)IsStream; // Stream bit - MIB.buildInstr(AArch64::G_PREFETCH).addImm(PrfOp).add(AddrVal); + MIB.buildInstr(AArch64::G_AARCH64_PREFETCH).addImm(PrfOp).add(AddrVal); MI.eraseFromParent(); return true; } @@ -1986,3 +1966,28 @@ bool AArch64LegalizerInfo::legalizeDynStackAlloc( MI.eraseFromParent(); return true; } + +bool AArch64LegalizerInfo::legalizePrefetch(MachineInstr &MI, + LegalizerHelper &Helper) const { + MachineIRBuilder &MIB = Helper.MIRBuilder; + auto &AddrVal = MI.getOperand(0); + + int64_t IsWrite = MI.getOperand(1).getImm(); + int64_t Locality = MI.getOperand(2).getImm(); + int64_t IsData = MI.getOperand(3).getImm(); + + bool IsStream = Locality == 0; + if (Locality != 0) { + assert(Locality <= 3 && "Prefetch locality out-of-range"); + // The locality degree is the opposite of the cache speed. + // Put the number the other way around. + // The encoding starts at 0 for level 1 + Locality = 3 - Locality; + } + + unsigned PrfOp = (IsWrite << 4) | (!IsData << 3) | (Locality << 1) | IsStream; + + MIB.buildInstr(AArch64::G_AARCH64_PREFETCH).addImm(PrfOp).add(AddrVal); + MI.eraseFromParent(); + return true; +} diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h index 6fd859d334cd..19f77baa77f8 100644 --- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h +++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.h @@ -64,6 +64,7 @@ private: bool legalizeExtractVectorElt(MachineInstr &MI, MachineRegisterInfo &MRI, LegalizerHelper &Helper) const; bool legalizeDynStackAlloc(MachineInstr &MI, LegalizerHelper &Helper) const; + bool legalizePrefetch(MachineInstr &MI, LegalizerHelper &Helper) const; const AArch64Subtarget *ST; }; } // End llvm namespace. diff --git a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp index c7ff14c252f1..a6900b8963bb 100644 --- a/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp +++ b/llvm/lib/Target/AArch64/MCTargetDesc/AArch64AsmBackend.cpp @@ -100,7 +100,8 @@ public: unsigned getFixupKindContainereSizeInBytes(unsigned Kind) const; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; }; } // end anonymous namespace @@ -499,7 +500,8 @@ bool AArch64AsmBackend::writeNopData(raw_ostream &OS, uint64_t Count, bool AArch64AsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { unsigned Kind = Fixup.getKind(); if (Kind >= FirstLiteralRelocationKind) return true; diff --git a/llvm/lib/Target/AArch64/SMEABIPass.cpp b/llvm/lib/Target/AArch64/SMEABIPass.cpp index c813d92ec85b..3315171798d9 100644 --- a/llvm/lib/Target/AArch64/SMEABIPass.cpp +++ b/llvm/lib/Target/AArch64/SMEABIPass.cpp @@ -15,7 +15,6 @@ #include "AArch64.h" #include "Utils/AArch64BaseInfo.h" #include "Utils/AArch64SMEAttributes.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringRef.h" #include "llvm/IR/Constants.h" #include "llvm/IR/IRBuilder.h" diff --git a/llvm/lib/Target/AMDGPU/AMDGPU.h b/llvm/lib/Target/AMDGPU/AMDGPU.h index 323560a46f31..1b75607e1dc3 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPU.h +++ b/llvm/lib/Target/AMDGPU/AMDGPU.h @@ -12,12 +12,12 @@ #include "llvm/IR/PassManager.h" #include "llvm/Pass.h" +#include "llvm/Support/AMDGPUAddrSpace.h" #include "llvm/Support/CodeGen.h" namespace llvm { class AMDGPUTargetMachine; -class GCNTargetMachine; class TargetMachine; // GlobalISel passes @@ -390,72 +390,6 @@ enum TargetIndex { TI_SCRATCH_RSRC_DWORD2, TI_SCRATCH_RSRC_DWORD3 }; -} - -/// OpenCL uses address spaces to differentiate between -/// various memory regions on the hardware. On the CPU -/// all of the address spaces point to the same memory, -/// however on the GPU, each address space points to -/// a separate piece of memory that is unique from other -/// memory locations. -namespace AMDGPUAS { -enum : unsigned { - // The maximum value for flat, generic, local, private, constant and region. - MAX_AMDGPU_ADDRESS = 8, - - FLAT_ADDRESS = 0, ///< Address space for flat memory. - GLOBAL_ADDRESS = 1, ///< Address space for global memory (RAT0, VTX0). - REGION_ADDRESS = 2, ///< Address space for region memory. (GDS) - - CONSTANT_ADDRESS = 4, ///< Address space for constant memory (VTX2). - LOCAL_ADDRESS = 3, ///< Address space for local memory. - PRIVATE_ADDRESS = 5, ///< Address space for private memory. - - CONSTANT_ADDRESS_32BIT = 6, ///< Address space for 32-bit constant memory. - - BUFFER_FAT_POINTER = 7, ///< Address space for 160-bit buffer fat pointers. - ///< Not used in backend. - - BUFFER_RESOURCE = 8, ///< Address space for 128-bit buffer resources. - - /// Internal address spaces. Can be freely renumbered. - STREAMOUT_REGISTER = 128, ///< Address space for GS NGG Streamout registers. - /// end Internal address spaces. - - /// Address space for direct addressable parameter memory (CONST0). - PARAM_D_ADDRESS = 6, - /// Address space for indirect addressable parameter memory (VTX1). - PARAM_I_ADDRESS = 7, - - // Do not re-order the CONSTANT_BUFFER_* enums. Several places depend on - // this order to be able to dynamically index a constant buffer, for - // example: - // - // ConstantBufferAS = CONSTANT_BUFFER_0 + CBIdx - - CONSTANT_BUFFER_0 = 8, - CONSTANT_BUFFER_1 = 9, - CONSTANT_BUFFER_2 = 10, - CONSTANT_BUFFER_3 = 11, - CONSTANT_BUFFER_4 = 12, - CONSTANT_BUFFER_5 = 13, - CONSTANT_BUFFER_6 = 14, - CONSTANT_BUFFER_7 = 15, - CONSTANT_BUFFER_8 = 16, - CONSTANT_BUFFER_9 = 17, - CONSTANT_BUFFER_10 = 18, - CONSTANT_BUFFER_11 = 19, - CONSTANT_BUFFER_12 = 20, - CONSTANT_BUFFER_13 = 21, - CONSTANT_BUFFER_14 = 22, - CONSTANT_BUFFER_15 = 23, - - // Some places use this if the address space can't be determined. - UNKNOWN_ADDRESS_SPACE = ~0u, -}; -} - -namespace AMDGPU { // FIXME: Missing constant_32bit inline bool isFlatGlobalAddrSpace(unsigned AS) { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h index 48d030d64f82..5ee888d9db00 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUGlobalISelUtils.h @@ -9,7 +9,6 @@ #ifndef LLVM_LIB_TARGET_AMDGPU_AMDGPUGLOBALISELUTILS_H #define LLVM_LIB_TARGET_AMDGPU_AMDGPUGLOBALISELUTILS_H -#include "llvm/ADT/ArrayRef.h" #include "llvm/CodeGen/Register.h" #include diff --git a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h index 18a7b5d7a963..6d6bd86711b1 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUHSAMetadataStreamer.h @@ -30,7 +30,6 @@ class MDNode; class Module; struct SIProgramInfo; class Type; -class GCNSubtarget; namespace AMDGPU { diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp index 0b2bb98738be..0a17b1536040 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.cpp @@ -345,13 +345,13 @@ class PipelineSolver { // return the number of edges missed. int addEdges(SmallVectorImpl &SyncPipeline, SUnit *SU, int SGID, std::vector> &AddedEdges); - // Link the pipeline as if \p SU was in the SchedGroup with ID \p SGID. It - // returns the cost (in terms of missed pipeline edges), and tracks the edges - // added in \p AddedEdges + /// Link the pipeline as if \p SU was in the SchedGroup with ID \p SGID. It + /// returns the cost (in terms of missed pipeline edges), and tracks the edges + /// added in \p AddedEdges template int linkSUnit(SUnit *SU, int SGID, std::vector> &AddedEdges, T I, T E); - // Remove the edges passed via \p AddedEdges + /// Remove the edges passed via \p AddedEdges void removeEdges(const std::vector> &AddedEdges); // Convert the passed in maps to arrays for bidirectional iterators void convertSyncMapsToArrays(); @@ -847,11 +847,11 @@ protected: const SIInstrInfo *TII; public: - // Add SchedGroups to \p Pipeline to implement this Strategy. + /// Add SchedGroups to \p SyncedSchedGroups to implement this Strategy. virtual void applyIGLPStrategy( DenseMap &SyncedInstrs, DenseMap> &SyncedSchedGroups, - bool IsPostRA) = 0; + bool IsReentry) = 0; // Returns true if this strategy should be applied to a ScheduleDAG. virtual bool shouldApplyStrategy(ScheduleDAGInstrs *DAG) = 0; @@ -870,7 +870,7 @@ public: void applyIGLPStrategy( DenseMap &SyncedInstrs, DenseMap> &SyncedSchedGroups, - bool IsPostRA) override; + bool IsReentry) override; bool shouldApplyStrategy(ScheduleDAGInstrs *DAG) override { return true; } @@ -883,7 +883,7 @@ public: void MFMASmallGemmOpt::applyIGLPStrategy( DenseMap &SyncedInstrs, DenseMap> &SyncedSchedGroups, - bool IsPostRA) { + bool IsReentry) { // Count the number of MFMA instructions. unsigned MFMACount = 0; for (const MachineInstr &I : *DAG) @@ -1045,8 +1045,8 @@ private: : InstructionRule(TII, SGID, NeedsCache) {} }; - // Whether the SU shares a V_PERM predecessor with any SU in the SchedGroup - // that is /p Distance steps away + /// Whether the SU shares a V_PERM predecessor with any SU in the SchedGroup + /// that is \p Distance steps away class SharesPredWithPrevNthGroup final : public InstructionRule { private: unsigned Distance = 1; @@ -1100,7 +1100,7 @@ public: void applyIGLPStrategy( DenseMap &SyncedInstrs, DenseMap> &SyncedSchedGroups, - bool IsPostRA) override; + bool IsReentry) override; bool shouldApplyStrategy(ScheduleDAGInstrs *DAG) override { return true; } @@ -1117,12 +1117,12 @@ static unsigned DSWWithSharedVMEMCount = 0; void MFMASmallGemmSingleWaveOpt::applyIGLPStrategy( DenseMap &SyncedInstrs, DenseMap> &SyncedSchedGroups, - bool IsPostRA) { + bool IsReentry) { unsigned MFMACount = 0; unsigned DSRCount = 0; - assert((IsPostRA || (DSWCount == 0 && DSWWithPermCount == 0 && - DSWWithSharedVMEMCount == 0)) && + assert((IsReentry || (DSWCount == 0 && DSWWithPermCount == 0 && + DSWWithSharedVMEMCount == 0)) && "DSWCounters should be zero in pre-RA scheduling!"); SmallVector DSWithPerms; for (auto &SU : DAG->SUnits) { @@ -1132,7 +1132,7 @@ void MFMASmallGemmSingleWaveOpt::applyIGLPStrategy( else if (TII->isDS(*I)) { if (I->mayLoad()) ++DSRCount; - else if (I->mayStore() && !IsPostRA) { + else if (I->mayStore() && !IsReentry) { ++DSWCount; for (auto Pred : SU.Preds) { if (Pred.getSUnit()->getInstr()->getOpcode() == @@ -1145,7 +1145,7 @@ void MFMASmallGemmSingleWaveOpt::applyIGLPStrategy( } } - if (!IsPostRA) { + if (!IsReentry) { DSWWithPermCount = DSWithPerms.size(); auto I = DSWithPerms.begin(); auto E = DSWithPerms.end(); @@ -1412,11 +1412,11 @@ public: // first created SchedGroup first. bool IsBottomUp = 1; - // Whether the mutation is being applied to post RA scheduling - bool IsPostRA = false; + // Whether or not this is a reentry into the IGroupLPDAGMutation. + bool IsReentry = false; IGroupLPDAGMutation() = default; - IGroupLPDAGMutation(bool IsPostRA) : IsPostRA(IsPostRA) {} + IGroupLPDAGMutation(bool IsReentry) : IsReentry(IsReentry) {} }; unsigned SchedGroup::NumSchedGroups = 0; @@ -1704,7 +1704,7 @@ void IGroupLPDAGMutation::initIGLPOpt(SUnit &SU) { auto S = createIGLPStrategy(StrategyID, DAG, TII); if (S->shouldApplyStrategy(DAG)) { IsBottomUp = S->IsBottomUp; - S->applyIGLPStrategy(SyncedInstrs, SyncedSchedGroups, IsPostRA); + S->applyIGLPStrategy(SyncedInstrs, SyncedSchedGroups, IsReentry); } } @@ -1712,8 +1712,13 @@ void IGroupLPDAGMutation::initIGLPOpt(SUnit &SU) { namespace llvm { -std::unique_ptr createIGroupLPDAGMutation(bool IsPostRA) { - return std::make_unique(IsPostRA); +/// \p IsReentry specifes whether or not this is a reentry into the +/// IGroupLPDAGMutation. Since there may be multiple scheduling passes on the +/// same scheduling region (e.g. pre and post-RA scheduling / multiple +/// scheduling "phases"), we can reenter this mutation framework more than once +/// for a given region. +std::unique_ptr createIGroupLPDAGMutation(bool IsReentry) { + return std::make_unique(IsReentry); } } // end namespace llvm diff --git a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h index eee2a48de396..3ec8be4f8892 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUIGroupLP.h @@ -14,7 +14,7 @@ namespace llvm { -std::unique_ptr createIGroupLPDAGMutation(bool IsPostRA); +std::unique_ptr createIGroupLPDAGMutation(bool IsReentry); } // namespace llvm diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index fb8d685409e4..a6d1da94b890 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -2334,9 +2334,8 @@ static SDValue combineBallotPattern(SDValue VCMP, bool &Negate) { // Note that ballot doesn't use SETEQ condition but its easy to support it // here for completeness, so in this case Negate is set true on return. auto VCMP_CC = cast(VCMP.getOperand(2))->get(); - auto *VCMP_CRHS = dyn_cast(VCMP.getOperand(1)); - if ((VCMP_CC == ISD::SETEQ || VCMP_CC == ISD::SETNE) && VCMP_CRHS && - VCMP_CRHS->isZero()) { + if ((VCMP_CC == ISD::SETEQ || VCMP_CC == ISD::SETNE) && + isNullConstant(VCMP.getOperand(1))) { auto Cond = VCMP.getOperand(0); if (ISD::isExtOpcode(Cond->getOpcode())) // Skip extension. @@ -2370,8 +2369,8 @@ void AMDGPUDAGToDAGISel::SelectBRCOND(SDNode *N) { Cond->getOperand(0)->getOpcode() == AMDGPUISD::SETCC) { SDValue VCMP = Cond->getOperand(0); auto CC = cast(Cond->getOperand(2))->get(); - auto *CRHS = dyn_cast(Cond->getOperand(1)); - if ((CC == ISD::SETEQ || CC == ISD::SETNE) && CRHS && CRHS->isZero() && + if ((CC == ISD::SETEQ || CC == ISD::SETNE) && + isNullConstant(Cond->getOperand(1)) && // TODO: make condition below an assert after fixing ballot bitwidth. VCMP.getValueType().getSizeInBits() == ST->getWavefrontSize()) { // %VCMP = i(WaveSize) AMDGPUISD::SETCC ... diff --git a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h index 25c0b4953ab7..248ee26a47eb 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h +++ b/llvm/lib/Target/AMDGPU/AMDGPUMachineFunction.h @@ -20,7 +20,6 @@ namespace llvm { class AMDGPUSubtarget; -class GCNSubtarget; class AMDGPUMachineFunction : public MachineFunctionInfo { /// A map to keep track of local memory objects and their offsets within the diff --git a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp index 3175f6358a04..1f11beb71101 100644 --- a/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp +++ b/llvm/lib/Target/AMDGPU/Disassembler/AMDGPUDisassembler.cpp @@ -771,7 +771,7 @@ DecodeStatus AMDGPUDisassembler::getInstruction(MCInst &MI, uint64_t &Size, } DecodeStatus AMDGPUDisassembler::convertEXPInst(MCInst &MI) const { - if (STI.hasFeature(AMDGPU::FeatureGFX11)) { + if (STI.hasFeature(AMDGPU::FeatureGFX11Insts)) { // The MCInst still has these fields even though they are no longer encoded // in the GFX11 instruction. insertNamedMCOperand(MI, MCOperand::createImm(0), AMDGPU::OpName::vm); @@ -782,9 +782,13 @@ DecodeStatus AMDGPUDisassembler::convertEXPInst(MCInst &MI) const { DecodeStatus AMDGPUDisassembler::convertVINTERPInst(MCInst &MI) const { if (MI.getOpcode() == AMDGPU::V_INTERP_P10_F16_F32_inreg_gfx11 || + MI.getOpcode() == AMDGPU::V_INTERP_P10_F16_F32_inreg_gfx12 || MI.getOpcode() == AMDGPU::V_INTERP_P10_RTZ_F16_F32_inreg_gfx11 || + MI.getOpcode() == AMDGPU::V_INTERP_P10_RTZ_F16_F32_inreg_gfx12 || MI.getOpcode() == AMDGPU::V_INTERP_P2_F16_F32_inreg_gfx11 || - MI.getOpcode() == AMDGPU::V_INTERP_P2_RTZ_F16_F32_inreg_gfx11) { + MI.getOpcode() == AMDGPU::V_INTERP_P2_F16_F32_inreg_gfx12 || + MI.getOpcode() == AMDGPU::V_INTERP_P2_RTZ_F16_F32_inreg_gfx11 || + MI.getOpcode() == AMDGPU::V_INTERP_P2_RTZ_F16_F32_inreg_gfx12) { // The MCInst has this field that is not directly encoded in the // instruction. insertNamedMCOperand(MI, MCOperand::createImm(0), AMDGPU::OpName::op_sel); diff --git a/llvm/lib/Target/AMDGPU/EXPInstructions.td b/llvm/lib/Target/AMDGPU/EXPInstructions.td index 14ba01f0d67c..ff1d661ef6fe 100644 --- a/llvm/lib/Target/AMDGPU/EXPInstructions.td +++ b/llvm/lib/Target/AMDGPU/EXPInstructions.td @@ -41,8 +41,8 @@ class EXP_Real_ComprVM } // Real instruction with optional asm operand "row_en". -class EXP_Real_Row - : EXPCommon + : EXPCommon, SIMCInstr { let AsmMatchConverter = "cvtExp"; @@ -105,12 +105,12 @@ def EXP_gfx10 : EXP_Real_gfx10<0, "EXP">; def EXP_DONE_gfx10 : EXP_Real_gfx10<1, "EXP_DONE">; //===----------------------------------------------------------------------===// -// GFX11+ +// GFX11 //===----------------------------------------------------------------------===// class EXP_Real_gfx11 : EXP_Real_Row<_row, _done, pseudo, SIEncodingFamily.GFX11>, EXPe_Row { - let AssemblerPredicate = isGFX11Plus; + let AssemblerPredicate = isGFX11Only; let DecoderNamespace = "GFX11"; let row = _row; let done = _done; @@ -121,6 +121,24 @@ def EXP_DONE_gfx11 : EXP_Real_gfx11<0, 1, "EXP_DONE">; def EXP_ROW_gfx11 : EXP_Real_gfx11<1, 0, "EXP_ROW">; def EXP_ROW_DONE_gfx11 : EXP_Real_gfx11<1, 1, "EXP_ROW_DONE">; +//===----------------------------------------------------------------------===// +// GFX12+ +//===----------------------------------------------------------------------===// + +class VEXPORT_Real_gfx12 + : EXP_Real_Row<_row, _done, pseudo, SIEncodingFamily.GFX12, "export">, + EXPe_Row, MnemonicAlias<"exp", "export">, Requires<[isGFX12Plus]> { + let AssemblerPredicate = isGFX12Plus; + let DecoderNamespace = "GFX12"; + let row = _row; + let done = _done; +} + +def EXPORT_gfx12 : VEXPORT_Real_gfx12<0, 0, "EXP">; +def EXPORT_DONE_gfx12 : VEXPORT_Real_gfx12<0, 1, "EXP_DONE">; +def EXPORT_ROW_gfx12 : VEXPORT_Real_gfx12<1, 0, "EXP_ROW">; +def EXPORT_ROW_DONE_gfx12 : VEXPORT_Real_gfx12<1, 1, "EXP_ROW_DONE">; + //===----------------------------------------------------------------------===// // EXP Patterns //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp index 5ebf834377f2..fd8f0bebd3be 100644 --- a/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp +++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.cpp @@ -274,32 +274,42 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) { if (MI.isDebugInstr()) return; - auto DecrementDef = [this](const MachineOperand &MO) { + // Kill all defs. + GCNRegPressure DefPressure, ECDefPressure; + bool HasECDefs = false; + for (const MachineOperand &MO : MI.all_defs()) { + if (!MO.getReg().isVirtual()) + continue; + Register Reg = MO.getReg(); + LaneBitmask DefMask = getDefRegMask(MO, *MRI); + + // Treat a def as fully live at the moment of definition: keep a record. + if (MO.isEarlyClobber()) { + ECDefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI); + HasECDefs = true; + } else + DefPressure.inc(Reg, LaneBitmask::getNone(), DefMask, *MRI); + auto I = LiveRegs.find(Reg); if (I == LiveRegs.end()) - return; + continue; LaneBitmask &LiveMask = I->second; LaneBitmask PrevMask = LiveMask; - LiveMask &= ~getDefRegMask(MO, *MRI); + LiveMask &= ~DefMask; CurPressure.inc(Reg, PrevMask, LiveMask, *MRI); if (LiveMask.none()) LiveRegs.erase(I); - }; - - // Decrement non-early-clobber defs. - SmallVector EarlyClobberDefs; - for (const MachineOperand &MO : MI.all_defs()) { - if (!MO.getReg().isVirtual()) - continue; - if (!MO.isEarlyClobber()) - DecrementDef(MO); - else - EarlyClobberDefs.push_back(&MO); } - // Increment uses. + // Update MaxPressure with defs pressure. + DefPressure += CurPressure; + if (HasECDefs) + DefPressure += ECDefPressure; + MaxPressure = max(DefPressure, MaxPressure); + + // Make uses alive. SmallVector RegUses; collectVirtualRegUses(RegUses, MI, LIS, *MRI); for (const RegisterMaskPair &U : RegUses) { @@ -309,13 +319,9 @@ void GCNUpwardRPTracker::recede(const MachineInstr &MI) { CurPressure.inc(U.RegUnit, PrevMask, LiveMask, *MRI); } - // Point of maximum pressure: non-early-clobber defs are decremented and uses - // are incremented. - MaxPressure = max(CurPressure, MaxPressure); - - // Now decrement early clobber defs. - for (const MachineOperand *MO : EarlyClobberDefs) - DecrementDef(*MO); + // Update MaxPressure with uses plus early-clobber defs pressure. + MaxPressure = HasECDefs ? max(CurPressure + ECDefPressure, MaxPressure) + : max(CurPressure, MaxPressure); assert(CurPressure == getRegPressure(*MRI, LiveRegs)); } diff --git a/llvm/lib/Target/AMDGPU/GCNRegPressure.h b/llvm/lib/Target/AMDGPU/GCNRegPressure.h index e21bf10d795b..4100970fe1a9 100644 --- a/llvm/lib/Target/AMDGPU/GCNRegPressure.h +++ b/llvm/lib/Target/AMDGPU/GCNRegPressure.h @@ -85,6 +85,18 @@ struct GCNRegPressure { return !(*this == O); } + GCNRegPressure &operator+=(const GCNRegPressure &RHS) { + for (unsigned I = 0; I < TOTAL_KINDS; ++I) + Value[I] += RHS.Value[I]; + return *this; + } + + GCNRegPressure &operator-=(const GCNRegPressure &RHS) { + for (unsigned I = 0; I < TOTAL_KINDS; ++I) + Value[I] -= RHS.Value[I]; + return *this; + } + void dump() const; private: @@ -105,6 +117,20 @@ inline GCNRegPressure max(const GCNRegPressure &P1, const GCNRegPressure &P2) { return Res; } +inline GCNRegPressure operator+(const GCNRegPressure &P1, + const GCNRegPressure &P2) { + GCNRegPressure Sum = P1; + Sum += P2; + return Sum; +} + +inline GCNRegPressure operator-(const GCNRegPressure &P1, + const GCNRegPressure &P2) { + GCNRegPressure Diff = P1; + Diff -= P2; + return Diff; +} + class GCNRPTracker { public: using LiveRegSet = DenseMap; diff --git a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp index 6c044cae0d17..342d518f38bf 100644 --- a/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp +++ b/llvm/lib/Target/AMDGPU/GCNSchedStrategy.cpp @@ -853,7 +853,9 @@ bool GCNSchedStage::initGCNRegion() { StageID != GCNSchedStageID::UnclusteredHighRPReschedule) { SavedMutations.clear(); SavedMutations.swap(DAG.Mutations); - DAG.addMutation(createIGroupLPDAGMutation(/*IsPostRA=*/false)); + bool IsInitialStage = StageID == GCNSchedStageID::OccInitialSchedule || + StageID == GCNSchedStageID::ILPInitialSchedule; + DAG.addMutation(createIGroupLPDAGMutation(/*IsReentry=*/!IsInitialStage)); } return true; @@ -1567,7 +1569,7 @@ void GCNPostScheduleDAGMILive::schedule() { if (HasIGLPInstrs) { SavedMutations.clear(); SavedMutations.swap(Mutations); - addMutation(createIGroupLPDAGMutation(/*IsPostRA=*/true)); + addMutation(createIGroupLPDAGMutation(/*IsReentry=*/true)); } ScheduleDAGMI::schedule(); diff --git a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp index e18c04e62314..f91f36ed851b 100644 --- a/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp +++ b/llvm/lib/Target/AMDGPU/MCTargetDesc/AMDGPUAsmBackend.cpp @@ -53,7 +53,8 @@ public: std::optional getFixupKind(StringRef Name) const override; const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; }; } //End anonymous namespace @@ -192,7 +193,8 @@ const MCFixupKindInfo &AMDGPUAsmBackend::getFixupKindInfo( bool AMDGPUAsmBackend::shouldForceRelocation(const MCAssembler &, const MCFixup &Fixup, - const MCValue &) { + const MCValue &, + const MCSubtargetInfo *STI) { return Fixup.getKind() >= FirstLiteralRelocationKind; } diff --git a/llvm/lib/Target/AMDGPU/SIDefines.h b/llvm/lib/Target/AMDGPU/SIDefines.h index 47dc59e77dc4..29397e109706 100644 --- a/llvm/lib/Target/AMDGPU/SIDefines.h +++ b/llvm/lib/Target/AMDGPU/SIDefines.h @@ -413,8 +413,8 @@ enum Id { // Message ID, width(4) [3:0]. ID_DEALLOC_VGPRS_GFX11Plus = 3, // reused in GFX11 ID_SAVEWAVE = 4, // added in GFX8, removed in GFX11 - ID_STALL_WAVE_GEN = 5, // added in GFX9 - ID_HALT_WAVES = 6, // added in GFX9 + ID_STALL_WAVE_GEN = 5, // added in GFX9, removed in GFX12 + ID_HALT_WAVES = 6, // added in GFX9, removed in GFX12 ID_ORDERED_PS_DONE = 7, // added in GFX9, removed in GFX11 ID_EARLY_PRIM_DEALLOC = 8, // added in GFX9, removed in GFX10 ID_GS_ALLOC_REQ = 9, // added in GFX9 @@ -428,6 +428,7 @@ enum Id { // Message ID, width(4) [3:0]. ID_RTN_GET_REALTIME = 131, ID_RTN_SAVE_WAVE = 132, ID_RTN_GET_TBA = 133, + ID_RTN_GET_SE_AID_ID = 134, ID_MASK_PreGFX11_ = 0xF, ID_MASK_GFX11Plus_ = 0xFF diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp index 403efd6ffed3..23434d2de0fc 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUAsmUtils.cpp @@ -36,14 +36,15 @@ namespace SendMsg { // Disable lint checking for this block since it makes the table unreadable. // NOLINTBEGIN +// clang-format off const CustomOperand Msg[] = { {{""}}, {{"MSG_INTERRUPT"}, ID_INTERRUPT}, {{"MSG_GS"}, ID_GS_PreGFX11, isNotGFX11Plus}, {{"MSG_GS_DONE"}, ID_GS_DONE_PreGFX11, isNotGFX11Plus}, {{"MSG_SAVEWAVE"}, ID_SAVEWAVE, isGFX8_GFX9_GFX10}, - {{"MSG_STALL_WAVE_GEN"}, ID_STALL_WAVE_GEN, isGFX9Plus}, - {{"MSG_HALT_WAVES"}, ID_HALT_WAVES, isGFX9Plus}, + {{"MSG_STALL_WAVE_GEN"}, ID_STALL_WAVE_GEN, isGFX9_GFX10_GFX11}, + {{"MSG_HALT_WAVES"}, ID_HALT_WAVES, isGFX9_GFX10_GFX11}, {{"MSG_ORDERED_PS_DONE"}, ID_ORDERED_PS_DONE, isGFX9_GFX10}, {{"MSG_EARLY_PRIM_DEALLOC"}, ID_EARLY_PRIM_DEALLOC, isGFX9_GFX10}, {{"MSG_GS_ALLOC_REQ"}, ID_GS_ALLOC_REQ, isGFX9Plus}, @@ -59,7 +60,9 @@ const CustomOperand Msg[] = { {{"MSG_RTN_GET_REALTIME"}, ID_RTN_GET_REALTIME, isGFX11Plus}, {{"MSG_RTN_SAVE_WAVE"}, ID_RTN_SAVE_WAVE, isGFX11Plus}, {{"MSG_RTN_GET_TBA"}, ID_RTN_GET_TBA, isGFX11Plus}, + {{"MSG_RTN_GET_SE_AID_ID"}, ID_RTN_GET_SE_AID_ID, isGFX12Plus}, }; +// clang-format on // NOLINTEND const int MSG_SIZE = static_cast( diff --git a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h index 7c6ed01a1cd4..e42b27f8e09e 100644 --- a/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h +++ b/llvm/lib/Target/AMDGPU/Utils/AMDGPUMemoryUtils.h @@ -14,12 +14,10 @@ namespace llvm { struct Align; class AAResults; class DataLayout; -class Function; class GlobalVariable; class LoadInst; class MemoryDef; class MemorySSA; -class Module; class Value; namespace AMDGPU { diff --git a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td index 7d03150bf5b1..fc563b7493ad 100644 --- a/llvm/lib/Target/AMDGPU/VINTERPInstructions.td +++ b/llvm/lib/Target/AMDGPU/VINTERPInstructions.td @@ -10,7 +10,7 @@ // VINTERP encoding //===----------------------------------------------------------------------===// -class VINTERPe_gfx11 op, VOPProfile P> : Enc64 { +class VINTERPe : Enc64 { bits<8> vdst; bits<4> src0_modifiers; bits<9> src0; @@ -31,7 +31,6 @@ class VINTERPe_gfx11 op, VOPProfile P> : Enc64 { let Inst{13} = !if(P.HasOpSel, src2_modifiers{2}, 0); // op_sel(2) let Inst{14} = !if(P.HasOpSel, src0_modifiers{3}, 0); // op_sel(3) let Inst{15} = clamp; - let Inst{22-16} = op; let Inst{40-32} = src0; let Inst{49-41} = src1; let Inst{58-50} = src2; @@ -40,6 +39,14 @@ class VINTERPe_gfx11 op, VOPProfile P> : Enc64 { let Inst{63} = src2_modifiers{0}; // neg(2) } +class VINTERPe_gfx11 op, VOPProfile P> : VINTERPe

{ + let Inst{22-16} = op; +} + +class VINTERPe_gfx12 op, VOPProfile P> : VINTERPe

{ + let Inst{20-16} = op{4-0}; +} + //===----------------------------------------------------------------------===// // VOP3 VINTERP //===----------------------------------------------------------------------===// @@ -171,17 +178,28 @@ defm : VInterpF16Pat op> { +multiclass VINTERP_Real_gfx11 op> { + let AssemblerPredicate = isGFX11Only, DecoderNamespace = "GFX11" in { def _gfx11 : VINTERP_Real(NAME), SIEncodingFamily.GFX11>, VINTERPe_gfx11(NAME).Pfl>; } } -defm V_INTERP_P10_F32_inreg : VINTERP_Real_gfx11<0x000>; -defm V_INTERP_P2_F32_inreg : VINTERP_Real_gfx11<0x001>; -defm V_INTERP_P10_F16_F32_inreg : VINTERP_Real_gfx11<0x002>; -defm V_INTERP_P2_F16_F32_inreg : VINTERP_Real_gfx11<0x003>; -defm V_INTERP_P10_RTZ_F16_F32_inreg : VINTERP_Real_gfx11<0x004>; -defm V_INTERP_P2_RTZ_F16_F32_inreg : VINTERP_Real_gfx11<0x005>; +multiclass VINTERP_Real_gfx12 op> { + let AssemblerPredicate = isGFX12Only, DecoderNamespace = "GFX12" in { + def _gfx12 : + VINTERP_Real(NAME), SIEncodingFamily.GFX12>, + VINTERPe_gfx12(NAME).Pfl>; + } +} + +multiclass VINTERP_Real_gfx11_gfx12 op> : + VINTERP_Real_gfx11, VINTERP_Real_gfx12; + +defm V_INTERP_P10_F32_inreg : VINTERP_Real_gfx11_gfx12<0x000>; +defm V_INTERP_P2_F32_inreg : VINTERP_Real_gfx11_gfx12<0x001>; +defm V_INTERP_P10_F16_F32_inreg : VINTERP_Real_gfx11_gfx12<0x002>; +defm V_INTERP_P2_F16_F32_inreg : VINTERP_Real_gfx11_gfx12<0x003>; +defm V_INTERP_P10_RTZ_F16_F32_inreg : VINTERP_Real_gfx11_gfx12<0x004>; +defm V_INTERP_P2_RTZ_F16_F32_inreg : VINTERP_Real_gfx11_gfx12<0x005>; diff --git a/llvm/lib/Target/AMDGPU/VOP3Instructions.td b/llvm/lib/Target/AMDGPU/VOP3Instructions.td index 114d33b07786..a73042f2e411 100644 --- a/llvm/lib/Target/AMDGPU/VOP3Instructions.td +++ b/llvm/lib/Target/AMDGPU/VOP3Instructions.td @@ -678,11 +678,22 @@ multiclass IMAD32_Pats { >; } +// Handle cases where amdgpu-codegenprepare-mul24 made a mul24 instead of a normal mul. +// We need to separate this because otherwise OtherPredicates would be overriden. +class IMAD32_Mul24_Pat: GCNPat < + (i64 (add (i64 (AMDGPUmul_u24 i32:$src0, i32:$src1)), i64:$src2)), + (inst $src0, $src1, $src2, 0 /* clamp */) + >; + // exclude pre-GFX9 where it was slow -let OtherPredicates = [HasNotMADIntraFwdBug], SubtargetPredicate = isGFX9Plus in +let OtherPredicates = [HasNotMADIntraFwdBug], SubtargetPredicate = isGFX9Plus in { defm : IMAD32_Pats; -let OtherPredicates = [HasMADIntraFwdBug], SubtargetPredicate = isGFX11Only in + def : IMAD32_Mul24_Pat; +} +let OtherPredicates = [HasMADIntraFwdBug], SubtargetPredicate = isGFX11Only in { defm : IMAD32_Pats; + def : IMAD32_Mul24_Pat; +} def VOP3_PERMLANE_Profile : VOP3_Profile, VOP3_OPSEL> { let InsVOP3OpSel = (ins IntOpSelMods:$src0_modifiers, VRegSrc_32:$src0, diff --git a/llvm/lib/Target/ARM/ARM.td b/llvm/lib/Target/ARM/ARM.td index bf64ecf64964..97d1444a553e 100644 --- a/llvm/lib/Target/ARM/ARM.td +++ b/llvm/lib/Target/ARM/ARM.td @@ -1519,6 +1519,17 @@ def : ProcessorModel<"cortex-m85", CortexM85Model, [ARMv81mMainline, FeatureUseMISched, HasMVEFloatOps]>; +def : ProcessorModel<"cortex-m52", CortexM55Model, [ARMv81mMainline, + FeatureDSP, + FeatureFPARMv8_D16, + FeatureHasNoBranchPredictor, + FeaturePACBTI, + FeatureUseMISched, + FeaturePrefLoopAlign32, + FeatureHasSlowFPVMLx, + FeatureMVEVectorCostFactor1, + HasMVEFloatOps]>; + def : ProcNoItin<"cortex-a32", [ARMv8a, FeatureHWDivThumb, FeatureHWDivARM, @@ -1651,7 +1662,7 @@ def : ProcNoItin<"neoverse-n1", [ARMv82a, FeatureCRC, FeatureDotProd]>; -def : ProcNoItin<"neoverse-n2", [ARMv85a, +def : ProcNoItin<"neoverse-n2", [ARMv9a, FeatureBF16, FeatureMatMulInt8]>; diff --git a/llvm/lib/Target/ARM/ARMSubtarget.cpp b/llvm/lib/Target/ARM/ARMSubtarget.cpp index 1505e9214050..922fa93226f2 100644 --- a/llvm/lib/Target/ARM/ARMSubtarget.cpp +++ b/llvm/lib/Target/ARM/ARMSubtarget.cpp @@ -298,6 +298,7 @@ void ARMSubtarget::initSubtargetFeatures(StringRef CPU, StringRef FS) { case CortexM3: case CortexM7: case CortexR52: + case CortexM52: case CortexX1: case CortexX1C: break; diff --git a/llvm/lib/Target/ARM/ARMSubtarget.h b/llvm/lib/Target/ARM/ARMSubtarget.h index eb5bf25f2a38..43b4123a1b55 100644 --- a/llvm/lib/Target/ARM/ARMSubtarget.h +++ b/llvm/lib/Target/ARM/ARMSubtarget.h @@ -32,6 +32,7 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Target/TargetOptions.h" #include "llvm/TargetParser/Triple.h" +#include #include #include @@ -71,6 +72,7 @@ protected: CortexA9, CortexM3, CortexM7, + CortexM52, CortexR4, CortexR4F, CortexR5, diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp index 41b3c6005231..534434fef5ac 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.cpp @@ -908,7 +908,8 @@ unsigned ARMAsmBackend::adjustFixupValue(const MCAssembler &Asm, bool ARMAsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { const MCSymbolRefExpr *A = Target.getSymA(); const MCSymbol *Sym = A ? &A->getSymbol() : nullptr; const unsigned FixupKind = Fixup.getKind(); diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h index 40d111b79706..328eed9b0ec4 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMAsmBackend.h @@ -36,7 +36,8 @@ public: const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; unsigned adjustFixupValue(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, uint64_t Value, diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp index 62404f7add48..c62d17fd427a 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMUnwindOpAsm.cpp @@ -12,9 +12,9 @@ //===----------------------------------------------------------------------===// #include "ARMUnwindOpAsm.h" +#include "llvm/ADT/bit.h" #include "llvm/Support/ARMEHABI.h" #include "llvm/Support/LEB128.h" -#include "llvm/Support/MathExtras.h" #include using namespace llvm; diff --git a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp index 31a814900ca5..c4427948d3b8 100644 --- a/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp +++ b/llvm/lib/Target/ARM/MCTargetDesc/ARMWinCOFFObjectWriter.cpp @@ -19,7 +19,6 @@ #include "llvm/MC/MCWinCOFFObjectWriter.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/raw_ostream.h" -#include using namespace llvm; diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp index c94469c8d9f3..d520880d73bb 100644 --- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp +++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.cpp @@ -507,7 +507,8 @@ bool AVRAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count, bool AVRAsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { switch ((unsigned)Fixup.getKind()) { default: return Fixup.getKind() >= FirstLiteralRelocationKind; diff --git a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h index 3081fe1fd58c..023660f0ff14 100644 --- a/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h +++ b/llvm/lib/Target/AVR/MCTargetDesc/AVRAsmBackend.h @@ -60,7 +60,8 @@ public: const MCSubtargetInfo *STI) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; private: Triple::OSType OSType; diff --git a/llvm/lib/Target/BPF/BPF.h b/llvm/lib/Target/BPF/BPF.h index 436cd62c2581..5c77d183e1ef 100644 --- a/llvm/lib/Target/BPF/BPF.h +++ b/llvm/lib/Target/BPF/BPF.h @@ -16,7 +16,10 @@ #include "llvm/Target/TargetMachine.h" namespace llvm { +class BPFRegisterBankInfo; +class BPFSubtarget; class BPFTargetMachine; +class InstructionSelector; class PassRegistry; ModulePass *createBPFCheckAndAdjustIR(); @@ -27,6 +30,10 @@ FunctionPass *createBPFMIPeepholePass(); FunctionPass *createBPFMIPreEmitPeepholePass(); FunctionPass *createBPFMIPreEmitCheckingPass(); +InstructionSelector *createBPFInstructionSelector(const BPFTargetMachine &, + const BPFSubtarget &, + const BPFRegisterBankInfo &); + void initializeBPFCheckAndAdjustIRPass(PassRegistry&); void initializeBPFDAGToDAGISelPass(PassRegistry &); void initializeBPFMIPeepholePass(PassRegistry &); diff --git a/llvm/lib/Target/BPF/BPF.td b/llvm/lib/Target/BPF/BPF.td index 7f38fbdd8c5c..dff76ca07af5 100644 --- a/llvm/lib/Target/BPF/BPF.td +++ b/llvm/lib/Target/BPF/BPF.td @@ -11,6 +11,7 @@ include "llvm/Target/Target.td" include "BPFRegisterInfo.td" include "BPFCallingConv.td" include "BPFInstrInfo.td" +include "GISel/BPFRegisterBanks.td" def BPFInstrInfo : InstrInfo; diff --git a/llvm/lib/Target/BPF/BPFInstrInfo.td b/llvm/lib/Target/BPF/BPFInstrInfo.td index 5972c9d49c51..7d443a344901 100644 --- a/llvm/lib/Target/BPF/BPFInstrInfo.td +++ b/llvm/lib/Target/BPF/BPFInstrInfo.td @@ -49,8 +49,8 @@ def BPFWrapper : SDNode<"BPFISD::Wrapper", SDT_BPFWrapper>; def BPFmemcpy : SDNode<"BPFISD::MEMCPY", SDT_BPFMEMCPY, [SDNPHasChain, SDNPInGlue, SDNPOutGlue, SDNPMayStore, SDNPMayLoad]>; -def BPFIsLittleEndian : Predicate<"CurDAG->getDataLayout().isLittleEndian()">; -def BPFIsBigEndian : Predicate<"!CurDAG->getDataLayout().isLittleEndian()">; +def BPFIsLittleEndian : Predicate<"Subtarget->isLittleEndian()">; +def BPFIsBigEndian : Predicate<"!Subtarget->isLittleEndian()">; def BPFHasALU32 : Predicate<"Subtarget->getHasAlu32()">; def BPFNoALU32 : Predicate<"!Subtarget->getHasAlu32()">; def BPFHasLdsx : Predicate<"Subtarget->hasLdsx()">; diff --git a/llvm/lib/Target/BPF/BPFSubtarget.cpp b/llvm/lib/Target/BPF/BPFSubtarget.cpp index ce02c831828e..9a8e42f32371 100644 --- a/llvm/lib/Target/BPF/BPFSubtarget.cpp +++ b/llvm/lib/Target/BPF/BPFSubtarget.cpp @@ -12,6 +12,10 @@ #include "BPFSubtarget.h" #include "BPF.h" +#include "BPFTargetMachine.h" +#include "GISel/BPFCallLowering.h" +#include "GISel/BPFLegalizerInfo.h" +#include "GISel/BPFRegisterBankInfo.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/TargetParser/Host.h" @@ -93,4 +97,30 @@ BPFSubtarget::BPFSubtarget(const Triple &TT, const std::string &CPU, const std::string &FS, const TargetMachine &TM) : BPFGenSubtargetInfo(TT, CPU, /*TuneCPU*/ CPU, FS), FrameLowering(initializeSubtargetDependencies(CPU, FS)), - TLInfo(TM, *this) {} + TLInfo(TM, *this) { + IsLittleEndian = TT.isLittleEndian(); + + CallLoweringInfo.reset(new BPFCallLowering(*getTargetLowering())); + Legalizer.reset(new BPFLegalizerInfo(*this)); + auto *RBI = new BPFRegisterBankInfo(*getRegisterInfo()); + RegBankInfo.reset(RBI); + + InstSelector.reset(createBPFInstructionSelector( + *static_cast(&TM), *this, *RBI)); +} + +const CallLowering *BPFSubtarget::getCallLowering() const { + return CallLoweringInfo.get(); +} + +InstructionSelector *BPFSubtarget::getInstructionSelector() const { + return InstSelector.get(); +} + +const LegalizerInfo *BPFSubtarget::getLegalizerInfo() const { + return Legalizer.get(); +} + +const RegisterBankInfo *BPFSubtarget::getRegBankInfo() const { + return RegBankInfo.get(); +} diff --git a/llvm/lib/Target/BPF/BPFSubtarget.h b/llvm/lib/Target/BPF/BPFSubtarget.h index 6e81daa4d955..33747546eadc 100644 --- a/llvm/lib/Target/BPF/BPFSubtarget.h +++ b/llvm/lib/Target/BPF/BPFSubtarget.h @@ -16,7 +16,12 @@ #include "BPFFrameLowering.h" #include "BPFISelLowering.h" #include "BPFInstrInfo.h" +#include "BPFRegisterInfo.h" #include "BPFSelectionDAGInfo.h" +#include "llvm/CodeGen/GlobalISel/CallLowering.h" +#include "llvm/CodeGen/GlobalISel/InstructionSelector.h" +#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" +#include "llvm/CodeGen/RegisterBankInfo.h" #include "llvm/CodeGen/SelectionDAGTargetInfo.h" #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DataLayout.h" @@ -43,6 +48,8 @@ protected: // unused bool isDummyMode; + bool IsLittleEndian; + // whether the cpu supports jmp ext bool HasJmpExt; @@ -59,6 +66,11 @@ protected: // whether cpu v4 insns are enabled. bool HasLdsx, HasMovsx, HasBswap, HasSdivSmod, HasGotol, HasStoreImm; + std::unique_ptr CallLoweringInfo; + std::unique_ptr InstSelector; + std::unique_ptr Legalizer; + std::unique_ptr RegBankInfo; + public: // This constructor initializes the data members to match that // of the specified triple. @@ -81,6 +93,8 @@ public: bool hasGotol() const { return HasGotol; } bool hasStoreImm() const { return HasStoreImm; } + bool isLittleEndian() const { return IsLittleEndian; } + const BPFInstrInfo *getInstrInfo() const override { return &InstrInfo; } const BPFFrameLowering *getFrameLowering() const override { return &FrameLowering; @@ -91,9 +105,14 @@ public: const BPFSelectionDAGInfo *getSelectionDAGInfo() const override { return &TSInfo; } - const TargetRegisterInfo *getRegisterInfo() const override { + const BPFRegisterInfo *getRegisterInfo() const override { return &InstrInfo.getRegisterInfo(); } + + const CallLowering *getCallLowering() const override; + InstructionSelector *getInstructionSelector() const override; + const LegalizerInfo *getLegalizerInfo() const override; + const RegisterBankInfo *getRegBankInfo() const override; }; } // End llvm namespace diff --git a/llvm/lib/Target/BPF/BPFTargetMachine.cpp b/llvm/lib/Target/BPF/BPFTargetMachine.cpp index 65286c822c4b..ab0db576f7f7 100644 --- a/llvm/lib/Target/BPF/BPFTargetMachine.cpp +++ b/llvm/lib/Target/BPF/BPFTargetMachine.cpp @@ -15,10 +15,15 @@ #include "BPFTargetTransformInfo.h" #include "MCTargetDesc/BPFMCAsmInfo.h" #include "TargetInfo/BPFTargetInfo.h" +#include "llvm/CodeGen/GlobalISel/IRTranslator.h" +#include "llvm/CodeGen/GlobalISel/InstructionSelect.h" +#include "llvm/CodeGen/GlobalISel/Legalizer.h" +#include "llvm/CodeGen/GlobalISel/RegBankSelect.h" #include "llvm/CodeGen/Passes.h" #include "llvm/CodeGen/TargetLoweringObjectFileImpl.h" #include "llvm/CodeGen/TargetPassConfig.h" #include "llvm/IR/PassManager.h" +#include "llvm/InitializePasses.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/Support/FormattedStream.h" @@ -40,6 +45,7 @@ extern "C" LLVM_EXTERNAL_VISIBILITY void LLVMInitializeBPFTarget() { RegisterTargetMachine Z(getTheBPFTarget()); PassRegistry &PR = *PassRegistry::getPassRegistry(); + initializeGlobalISel(PR); initializeBPFCheckAndAdjustIRPass(PR); initializeBPFMIPeepholePass(PR); initializeBPFDAGToDAGISelPass(PR); @@ -90,6 +96,11 @@ public: bool addInstSelector() override; void addMachineSSAOptimization() override; void addPreEmitPass() override; + + bool addIRTranslator() override; + bool addLegalizeMachineIR() override; + bool addRegBankSelect() override; + bool addGlobalInstructionSelect() override; }; } @@ -174,3 +185,23 @@ void BPFPassConfig::addPreEmitPass() { if (!DisableMIPeephole) addPass(createBPFMIPreEmitPeepholePass()); } + +bool BPFPassConfig::addIRTranslator() { + addPass(new IRTranslator()); + return false; +} + +bool BPFPassConfig::addLegalizeMachineIR() { + addPass(new Legalizer()); + return false; +} + +bool BPFPassConfig::addRegBankSelect() { + addPass(new RegBankSelect()); + return false; +} + +bool BPFPassConfig::addGlobalInstructionSelect() { + addPass(new InstructionSelect(getOptLevel())); + return false; +} diff --git a/llvm/lib/Target/BPF/CMakeLists.txt b/llvm/lib/Target/BPF/CMakeLists.txt index 6a96394a6aee..d88e7ade40b9 100644 --- a/llvm/lib/Target/BPF/CMakeLists.txt +++ b/llvm/lib/Target/BPF/CMakeLists.txt @@ -11,10 +11,16 @@ tablegen(LLVM BPFGenInstrInfo.inc -gen-instr-info) tablegen(LLVM BPFGenMCCodeEmitter.inc -gen-emitter) tablegen(LLVM BPFGenRegisterInfo.inc -gen-register-info) tablegen(LLVM BPFGenSubtargetInfo.inc -gen-subtarget) +tablegen(LLVM BPFGenGlobalISel.inc -gen-global-isel) +tablegen(LLVM BPFGenRegisterBank.inc -gen-register-bank) add_public_tablegen_target(BPFCommonTableGen) add_llvm_target(BPFCodeGen + GISel/BPFCallLowering.cpp + GISel/BPFInstructionSelector.cpp + GISel/BPFRegisterBankInfo.cpp + GISel/BPFLegalizerInfo.cpp BPFAbstractMemberAccess.cpp BPFAdjustOpt.cpp BPFAsmPrinter.cpp @@ -44,6 +50,7 @@ add_llvm_target(BPFCodeGen CodeGen CodeGenTypes Core + GlobalISel IPO MC Scalar diff --git a/llvm/lib/Target/BPF/GISel/BPFCallLowering.cpp b/llvm/lib/Target/BPF/GISel/BPFCallLowering.cpp new file mode 100644 index 000000000000..3829a1a3151f --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFCallLowering.cpp @@ -0,0 +1,46 @@ +//===-- BPFCallLowering.cpp - Call lowering for GlobalISel ------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file implements the lowering of LLVM calls to machine code calls for +/// GlobalISel. +/// +//===----------------------------------------------------------------------===// + +#include "BPFCallLowering.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "bpf-call-lowering" + +using namespace llvm; + +BPFCallLowering::BPFCallLowering(const BPFTargetLowering &TLI) + : CallLowering(&TLI) {} + +bool BPFCallLowering::lowerReturn(MachineIRBuilder &MIRBuilder, + const Value *Val, ArrayRef VRegs, + FunctionLoweringInfo &FLI, + Register SwiftErrorVReg) const { + if (!VRegs.empty()) + return false; + MIRBuilder.buildInstr(BPF::RET); + return true; +} + +bool BPFCallLowering::lowerFormalArguments(MachineIRBuilder &MIRBuilder, + const Function &F, + ArrayRef> VRegs, + FunctionLoweringInfo &FLI) const { + return VRegs.empty(); +} + +bool BPFCallLowering::lowerCall(MachineIRBuilder &MIRBuilder, + CallLoweringInfo &Info) const { + return false; +} diff --git a/llvm/lib/Target/BPF/GISel/BPFCallLowering.h b/llvm/lib/Target/BPF/GISel/BPFCallLowering.h new file mode 100644 index 000000000000..0099d2048fe5 --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFCallLowering.h @@ -0,0 +1,39 @@ +//===-- BPFCallLowering.h - Call lowering for GlobalISel --------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file describes how to lower LLVM calls to machine code calls. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_BPF_GISEL_BPFCALLLOWERING_H +#define LLVM_LIB_TARGET_BPF_GISEL_BPFCALLLOWERING_H + +#include "BPFISelLowering.h" +#include "llvm/CodeGen/GlobalISel/CallLowering.h" +#include "llvm/IR/CallingConv.h" + +namespace llvm { + +class BPFTargetLowering; + +class BPFCallLowering : public CallLowering { +public: + BPFCallLowering(const BPFTargetLowering &TLI); + bool lowerReturn(MachineIRBuilder &MIRBuilder, const Value *Val, + ArrayRef VRegs, FunctionLoweringInfo &FLI, + Register SwiftErrorVReg) const override; + bool lowerFormalArguments(MachineIRBuilder &MIRBuilder, const Function &F, + ArrayRef> VRegs, + FunctionLoweringInfo &FLI) const override; + bool lowerCall(MachineIRBuilder &MIRBuilder, + CallLoweringInfo &Info) const override; +}; +} // namespace llvm + +#endif diff --git a/llvm/lib/Target/BPF/GISel/BPFInstructionSelector.cpp b/llvm/lib/Target/BPF/GISel/BPFInstructionSelector.cpp new file mode 100644 index 000000000000..1effeb7a57b1 --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFInstructionSelector.cpp @@ -0,0 +1,91 @@ +//===- BPFInstructionSelector.cpp --------------------------------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements the targeting of the InstructionSelector class for BPF. +//===----------------------------------------------------------------------===// + +#include "BPFInstrInfo.h" +#include "BPFRegisterBankInfo.h" +#include "BPFSubtarget.h" +#include "BPFTargetMachine.h" +#include "llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h" +#include "llvm/CodeGen/GlobalISel/InstructionSelector.h" +#include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/IR/IntrinsicsBPF.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "bpf-gisel" + +using namespace llvm; + +namespace { + +#define GET_GLOBALISEL_PREDICATE_BITSET +#include "BPFGenGlobalISel.inc" +#undef GET_GLOBALISEL_PREDICATE_BITSET + +class BPFInstructionSelector : public InstructionSelector { +public: + BPFInstructionSelector(const BPFTargetMachine &TM, const BPFSubtarget &STI, + const BPFRegisterBankInfo &RBI); + + bool select(MachineInstr &I) override; + static const char *getName() { return DEBUG_TYPE; } + +private: + /// tblgen generated 'select' implementation that is used as the initial + /// selector for the patterns that do not require complex C++. + bool selectImpl(MachineInstr &I, CodeGenCoverage &CoverageInfo) const; + + const BPFInstrInfo &TII; + const BPFRegisterInfo &TRI; + const BPFRegisterBankInfo &RBI; + +#define GET_GLOBALISEL_PREDICATES_DECL +#include "BPFGenGlobalISel.inc" +#undef GET_GLOBALISEL_PREDICATES_DECL + +#define GET_GLOBALISEL_TEMPORARIES_DECL +#include "BPFGenGlobalISel.inc" +#undef GET_GLOBALISEL_TEMPORARIES_DECL +}; + +} // namespace + +#define GET_GLOBALISEL_IMPL +#include "BPFGenGlobalISel.inc" +#undef GET_GLOBALISEL_IMPL + +BPFInstructionSelector::BPFInstructionSelector(const BPFTargetMachine &TM, + const BPFSubtarget &STI, + const BPFRegisterBankInfo &RBI) + : TII(*STI.getInstrInfo()), TRI(*STI.getRegisterInfo()), RBI(RBI), +#define GET_GLOBALISEL_PREDICATES_INIT +#include "BPFGenGlobalISel.inc" +#undef GET_GLOBALISEL_PREDICATES_INIT +#define GET_GLOBALISEL_TEMPORARIES_INIT +#include "BPFGenGlobalISel.inc" +#undef GET_GLOBALISEL_TEMPORARIES_INIT +{ +} + +bool BPFInstructionSelector::select(MachineInstr &I) { + if (selectImpl(I, *CoverageInfo)) + return true; + return false; +} + +namespace llvm { +InstructionSelector * +createBPFInstructionSelector(const BPFTargetMachine &TM, + const BPFSubtarget &Subtarget, + const BPFRegisterBankInfo &RBI) { + return new BPFInstructionSelector(TM, Subtarget, RBI); +} +} // namespace llvm diff --git a/llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.cpp b/llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.cpp new file mode 100644 index 000000000000..04220c176376 --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.cpp @@ -0,0 +1,22 @@ +//===- BPFLegalizerInfo.h ----------------------------------------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements the targeting of the Machinelegalizer class for BPF +//===----------------------------------------------------------------------===// + +#include "BPFLegalizerInfo.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "bpf-legalinfo" + +using namespace llvm; +using namespace LegalizeActions; + +BPFLegalizerInfo::BPFLegalizerInfo(const BPFSubtarget &ST) { + getLegacyLegalizerInfo().computeTables(); +} diff --git a/llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.h b/llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.h new file mode 100644 index 000000000000..1704bc03144c --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFLegalizerInfo.h @@ -0,0 +1,28 @@ +//===- BPFLegalizerInfo.h ----------------------------------------*- C++ -*-==// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file declares the targeting of the Machinelegalizer class for BPF +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_BPF_GISEL_BPFMACHINELEGALIZER_H +#define LLVM_LIB_TARGET_BPF_GISEL_BPFMACHINELEGALIZER_H + +#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" + +namespace llvm { + +class BPFSubtarget; + +/// This class provides the information for the BPF target legalizer for +/// GlobalISel. +class BPFLegalizerInfo : public LegalizerInfo { +public: + BPFLegalizerInfo(const BPFSubtarget &ST); +}; +} // namespace llvm +#endif diff --git a/llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.cpp b/llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.cpp new file mode 100644 index 000000000000..f50e8f524a87 --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.cpp @@ -0,0 +1,25 @@ +//===- BPFRegisterBankInfo.cpp --------------------------------------------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// \file +/// This file implements the targeting of the RegisterBankInfo class for BPF +//===----------------------------------------------------------------------===// + +#include "BPFRegisterBankInfo.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/Support/Debug.h" + +#define DEBUG_TYPE "bpf-reg-bank-info" + +#define GET_TARGET_REGBANK_IMPL +#include "BPFGenRegisterBank.inc" + +using namespace llvm; + +BPFRegisterBankInfo::BPFRegisterBankInfo(const TargetRegisterInfo &TRI) + : BPFGenRegisterBankInfo() {} diff --git a/llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.h b/llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.h new file mode 100644 index 000000000000..82421916ca5e --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFRegisterBankInfo.h @@ -0,0 +1,39 @@ +//===-- BPFRegisterBankInfo.h -----------------------------------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// This file declares the targeting of the RegisterBankInfo class for BPF. +/// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_TARGET_BPF_GISEL_BPFREGISTERBANKINFO_H +#define LLVM_LIB_TARGET_BPF_GISEL_BPFREGISTERBANKINFO_H + +#include "MCTargetDesc/BPFMCTargetDesc.h" +#include "llvm/CodeGen/RegisterBankInfo.h" +#include "llvm/CodeGen/TargetRegisterInfo.h" + +#define GET_REGBANK_DECLARATIONS +#include "BPFGenRegisterBank.inc" + +namespace llvm { +class TargetRegisterInfo; + +class BPFGenRegisterBankInfo : public RegisterBankInfo { +protected: +#define GET_TARGET_REGBANK_CLASS +#include "BPFGenRegisterBank.inc" +}; + +class BPFRegisterBankInfo final : public BPFGenRegisterBankInfo { +public: + BPFRegisterBankInfo(const TargetRegisterInfo &TRI); +}; +} // namespace llvm + +#endif diff --git a/llvm/lib/Target/BPF/GISel/BPFRegisterBanks.td b/llvm/lib/Target/BPF/GISel/BPFRegisterBanks.td new file mode 100644 index 000000000000..af4af40a2537 --- /dev/null +++ b/llvm/lib/Target/BPF/GISel/BPFRegisterBanks.td @@ -0,0 +1,15 @@ +//===-- BPFRegisterBanks.td - Describe the BPF Banks -------*- tablegen -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +/// +/// \file +/// Define the BPF register banks used for GlobalISel. +/// +//===----------------------------------------------------------------------===// + +/// General Purpose Registers +def GPRRegBank : RegisterBank<"GPRB", [GPR]>; diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp b/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp index 15ab55f95e69..c266538bec73 100644 --- a/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp +++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFInstPrinter.cpp @@ -36,15 +36,16 @@ void BPFInstPrinter::printInst(const MCInst *MI, uint64_t Address, } static void printExpr(const MCExpr *Expr, raw_ostream &O) { -#ifndef NDEBUG const MCSymbolRefExpr *SRE; if (const MCBinaryExpr *BE = dyn_cast(Expr)) SRE = dyn_cast(BE->getLHS()); else SRE = dyn_cast(Expr); - assert(SRE && "Unexpected MCExpr type."); + if (!SRE) + report_fatal_error("Unexpected MCExpr type."); +#ifndef NDEBUG MCSymbolRefExpr::VariantKind Kind = SRE->getKind(); assert(Kind == MCSymbolRefExpr::VK_None); diff --git a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h index ea30e714a5b7..f12b79586baf 100644 --- a/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h +++ b/llvm/lib/Target/BPF/MCTargetDesc/BPFMCTargetDesc.h @@ -30,8 +30,7 @@ class MCSubtargetInfo; class MCTargetOptions; class Target; -MCCodeEmitter *createBPFMCCodeEmitter(const MCInstrInfo &MCII, - MCContext &Ctx); +MCCodeEmitter *createBPFMCCodeEmitter(const MCInstrInfo &MCII, MCContext &Ctx); MCCodeEmitter *createBPFbeMCCodeEmitter(const MCInstrInfo &MCII, MCContext &Ctx); @@ -43,7 +42,7 @@ MCAsmBackend *createBPFbeAsmBackend(const Target &T, const MCSubtargetInfo &STI, const MCTargetOptions &Options); std::unique_ptr createBPFELFObjectWriter(uint8_t OSABI); -} +} // namespace llvm // Defines symbolic names for BPF registers. This defines a mapping from // register name to register number. diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp index 76f5a5fc831f..bceb41a26745 100644 --- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp +++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.cpp @@ -262,7 +262,8 @@ bool CSKYAsmBackend::mayNeedRelaxation(const MCInst &Inst, bool CSKYAsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo * /*STI*/) { if (Fixup.getKind() >= FirstLiteralRelocationKind) return true; switch (Fixup.getTargetKind()) { diff --git a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h index 8a2f743bdee6..5fa0c8c01185 100644 --- a/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h +++ b/llvm/lib/Target/CSKY/MCTargetDesc/CSKYAsmBackend.h @@ -53,7 +53,8 @@ public: const MCSubtargetInfo *STI) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; std::unique_ptr createObjectTargetWriter() const override; diff --git a/llvm/lib/Target/DirectX/DXILResource.cpp b/llvm/lib/Target/DirectX/DXILResource.cpp index 0390a3f0a558..92306d907e05 100644 --- a/llvm/lib/Target/DirectX/DXILResource.cpp +++ b/llvm/lib/Target/DirectX/DXILResource.cpp @@ -233,9 +233,8 @@ void ResourceBase::print(raw_ostream &OS, StringRef IDPrefix, } UAVResource::UAVResource(uint32_t I, FrontendResource R) - : ResourceBase(I, R), - Shape(static_cast(R.getResourceKind())), - GloballyCoherent(false), HasCounter(false), IsROV(false), ExtProps() { + : ResourceBase(I, R), Shape(R.getResourceKind()), GloballyCoherent(false), + HasCounter(false), IsROV(R.getIsROV()), ExtProps() { parseSourceType(R.getSourceType()); } @@ -259,8 +258,6 @@ void UAVResource::print(raw_ostream &OS) const { // information we need to remove the source type string from here (See issue: // https://github.com/llvm/llvm-project/issues/57991). void UAVResource::parseSourceType(StringRef S) { - IsROV = S.startswith("RasterizerOrdered"); - S = S.substr(S.find("<") + 1); constexpr size_t PrefixLen = StringRef("vector<").size(); diff --git a/llvm/lib/Target/Hexagon/HexagonTargetMachine.h b/llvm/lib/Target/Hexagon/HexagonTargetMachine.h index 4ffd0fd89de6..dddd79ad1fcf 100644 --- a/llvm/lib/Target/Hexagon/HexagonTargetMachine.h +++ b/llvm/lib/Target/Hexagon/HexagonTargetMachine.h @@ -21,8 +21,6 @@ namespace llvm { -class Module; - class HexagonTargetMachine : public LLVMTargetMachine { std::unique_ptr TLOF; mutable StringMap> SubtargetMap; diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp index 76b4dc4e5afa..f9a0ba3608e6 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonAsmBackend.cpp @@ -202,7 +202,8 @@ public: } bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override { + const MCValue &Target, + const MCSubtargetInfo *STI) override { switch(Fixup.getTargetKind()) { default: llvm_unreachable("Unknown Fixup Kind!"); diff --git a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h index b83931eb88ac..42d91f559f51 100644 --- a/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h +++ b/llvm/lib/Target/Hexagon/MCTargetDesc/HexagonMCChecker.h @@ -17,7 +17,6 @@ #include "MCTargetDesc/HexagonMCInstrInfo.h" #include "MCTargetDesc/HexagonMCTargetDesc.h" #include "llvm/ADT/DenseMap.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/Support/SMLoc.h" #include #include diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index 400327373ad1..61ec63ff6c1c 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -247,6 +247,7 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Legal); setOperationAction(ISD::BUILD_VECTOR, VT, Custom); + setOperationAction(ISD::SETCC, VT, Legal); setOperationAction(ISD::VSELECT, VT, Legal); } for (MVT VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64}) { @@ -260,11 +261,19 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, setOperationAction({ISD::SHL, ISD::SRA, ISD::SRL}, VT, Legal); setOperationAction({ISD::CTPOP, ISD::CTLZ}, VT, Legal); setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Legal); + setCondCodeAction( + {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT, + Expand); } for (MVT VT : {MVT::v4f32, MVT::v2f64}) { setOperationAction({ISD::FADD, ISD::FSUB}, VT, Legal); setOperationAction({ISD::FMUL, ISD::FDIV}, VT, Legal); setOperationAction(ISD::FMA, VT, Legal); + setOperationAction(ISD::FSQRT, VT, Legal); + setOperationAction(ISD::FNEG, VT, Legal); + setCondCodeAction({ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT, + ISD::SETUGE, ISD::SETUGT}, + VT, Expand); } } @@ -280,6 +289,7 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Legal); setOperationAction(ISD::BUILD_VECTOR, VT, Custom); + setOperationAction(ISD::SETCC, VT, Legal); setOperationAction(ISD::VSELECT, VT, Legal); } for (MVT VT : {MVT::v4i64, MVT::v8i32, MVT::v16i16, MVT::v32i8}) { @@ -293,11 +303,19 @@ LoongArchTargetLowering::LoongArchTargetLowering(const TargetMachine &TM, setOperationAction({ISD::SHL, ISD::SRA, ISD::SRL}, VT, Legal); setOperationAction({ISD::CTPOP, ISD::CTLZ}, VT, Legal); setOperationAction({ISD::MULHS, ISD::MULHU}, VT, Legal); + setCondCodeAction( + {ISD::SETNE, ISD::SETGE, ISD::SETGT, ISD::SETUGE, ISD::SETUGT}, VT, + Expand); } for (MVT VT : {MVT::v8f32, MVT::v4f64}) { setOperationAction({ISD::FADD, ISD::FSUB}, VT, Legal); setOperationAction({ISD::FMUL, ISD::FDIV}, VT, Legal); setOperationAction(ISD::FMA, VT, Legal); + setOperationAction(ISD::FSQRT, VT, Legal); + setOperationAction(ISD::FNEG, VT, Legal); + setCondCodeAction({ISD::SETGE, ISD::SETGT, ISD::SETOGE, ISD::SETOGT, + ISD::SETUGE, ISD::SETUGT}, + VT, Expand); } } diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h index ec1e3351ac87..2d73a7394946 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h @@ -21,7 +21,6 @@ namespace llvm { class LoongArchSubtarget; -struct LoongArchRegisterInfo; namespace LoongArchISD { enum NodeType : unsigned { FIRST_NUMBER = ISD::BUILTIN_OP_END, diff --git a/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td index 0bd8db1bfdf0..ec6983d0f487 100644 --- a/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td +++ b/llvm/lib/Target/LoongArch/LoongArchLASXInstrInfo.td @@ -1092,6 +1092,13 @@ multiclass PatXr { (!cast(Inst#"_D") LASX256:$xj)>; } +multiclass PatXrF { + def : Pat<(v8f32 (OpNode (v8f32 LASX256:$xj))), + (!cast(Inst#"_S") LASX256:$xj)>; + def : Pat<(v4f64 (OpNode (v4f64 LASX256:$xj))), + (!cast(Inst#"_D") LASX256:$xj)>; +} + multiclass PatXrXr { def : Pat<(OpNode (v32i8 LASX256:$xj), (v32i8 LASX256:$xk)), (!cast(Inst#"_B") LASX256:$xj, LASX256:$xk)>; @@ -1184,6 +1191,65 @@ multiclass PatShiftXrUimm { (!cast(Inst#"_D") LASX256:$xj, uimm6:$imm)>; } +multiclass PatCCXrSimm5 { + def : Pat<(v32i8 (setcc (v32i8 LASX256:$xj), + (v32i8 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_B") LASX256:$xj, simm5:$imm)>; + def : Pat<(v16i16 (setcc (v16i16 LASX256:$xj), + (v16i16 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_H") LASX256:$xj, simm5:$imm)>; + def : Pat<(v8i32 (setcc (v8i32 LASX256:$xj), + (v8i32 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_W") LASX256:$xj, simm5:$imm)>; + def : Pat<(v4i64 (setcc (v4i64 LASX256:$xj), + (v4i64 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_D") LASX256:$xj, simm5:$imm)>; +} + +multiclass PatCCXrUimm5 { + def : Pat<(v32i8 (setcc (v32i8 LASX256:$xj), + (v32i8 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_BU") LASX256:$xj, uimm5:$imm)>; + def : Pat<(v16i16 (setcc (v16i16 LASX256:$xj), + (v16i16 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_HU") LASX256:$xj, uimm5:$imm)>; + def : Pat<(v8i32 (setcc (v8i32 LASX256:$xj), + (v8i32 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_WU") LASX256:$xj, uimm5:$imm)>; + def : Pat<(v4i64 (setcc (v4i64 LASX256:$xj), + (v4i64 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_DU") LASX256:$xj, uimm5:$imm)>; +} + +multiclass PatCCXrXr { + def : Pat<(v32i8 (setcc (v32i8 LASX256:$xj), (v32i8 LASX256:$xk), CC)), + (!cast(Inst#"_B") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v16i16 (setcc (v16i16 LASX256:$xj), (v16i16 LASX256:$xk), CC)), + (!cast(Inst#"_H") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v8i32 (setcc (v8i32 LASX256:$xj), (v8i32 LASX256:$xk), CC)), + (!cast(Inst#"_W") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v4i64 (setcc (v4i64 LASX256:$xj), (v4i64 LASX256:$xk), CC)), + (!cast(Inst#"_D") LASX256:$xj, LASX256:$xk)>; +} + +multiclass PatCCXrXrU { + def : Pat<(v32i8 (setcc (v32i8 LASX256:$xj), (v32i8 LASX256:$xk), CC)), + (!cast(Inst#"_BU") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v16i16 (setcc (v16i16 LASX256:$xj), (v16i16 LASX256:$xk), CC)), + (!cast(Inst#"_HU") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v8i32 (setcc (v8i32 LASX256:$xj), (v8i32 LASX256:$xk), CC)), + (!cast(Inst#"_WU") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v4i64 (setcc (v4i64 LASX256:$xj), (v4i64 LASX256:$xk), CC)), + (!cast(Inst#"_DU") LASX256:$xj, LASX256:$xk)>; +} + +multiclass PatCCXrXrF { + def : Pat<(v8i32 (setcc (v8f32 LASX256:$xj), (v8f32 LASX256:$xk), CC)), + (!cast(Inst#"_S") LASX256:$xj, LASX256:$xk)>; + def : Pat<(v4i64 (setcc (v4f64 LASX256:$xj), (v4f64 LASX256:$xk), CC)), + (!cast(Inst#"_D") LASX256:$xj, LASX256:$xk)>; +} + let Predicates = [HasExtLASX] in { // XVADD_{B/H/W/D} @@ -1389,6 +1455,83 @@ def : Pat<(fma v8f32:$xj, v8f32:$xk, v8f32:$xa), def : Pat<(fma v4f64:$xj, v4f64:$xk, v4f64:$xa), (XVFMADD_D v4f64:$xj, v4f64:$xk, v4f64:$xa)>; +// XVFMSUB_{S/D} +def : Pat<(fma v8f32:$xj, v8f32:$xk, (fneg v8f32:$xa)), + (XVFMSUB_S v8f32:$xj, v8f32:$xk, v8f32:$xa)>; +def : Pat<(fma v4f64:$xj, v4f64:$xk, (fneg v4f64:$xa)), + (XVFMSUB_D v4f64:$xj, v4f64:$xk, v4f64:$xa)>; + +// XVFNMADD_{S/D} +def : Pat<(fneg (fma v8f32:$xj, v8f32:$xk, v8f32:$xa)), + (XVFNMADD_S v8f32:$xj, v8f32:$xk, v8f32:$xa)>; +def : Pat<(fneg (fma v4f64:$xj, v4f64:$xk, v4f64:$xa)), + (XVFNMADD_D v4f64:$xj, v4f64:$xk, v4f64:$xa)>; +def : Pat<(fma_nsz (fneg v8f32:$xj), v8f32:$xk, (fneg v8f32:$xa)), + (XVFNMADD_S v8f32:$xj, v8f32:$xk, v8f32:$xa)>; +def : Pat<(fma_nsz (fneg v4f64:$xj), v4f64:$xk, (fneg v4f64:$xa)), + (XVFNMADD_D v4f64:$xj, v4f64:$xk, v4f64:$xa)>; + +// XVFNMSUB_{S/D} +def : Pat<(fneg (fma v8f32:$xj, v8f32:$xk, (fneg v8f32:$xa))), + (XVFNMSUB_S v8f32:$xj, v8f32:$xk, v8f32:$xa)>; +def : Pat<(fneg (fma v4f64:$xj, v4f64:$xk, (fneg v4f64:$xa))), + (XVFNMSUB_D v4f64:$xj, v4f64:$xk, v4f64:$xa)>; +def : Pat<(fma_nsz (fneg v8f32:$xj), v8f32:$xk, v8f32:$xa), + (XVFNMSUB_S v8f32:$xj, v8f32:$xk, v8f32:$xa)>; +def : Pat<(fma_nsz (fneg v4f64:$xj), v4f64:$xk, v4f64:$xa), + (XVFNMSUB_D v4f64:$xj, v4f64:$xk, v4f64:$xa)>; + +// XVFSQRT_{S/D} +defm : PatXrF; + +// XVRECIP_{S/D} +def : Pat<(fdiv vsplatf32_fpimm_eq_1, v8f32:$xj), + (XVFRECIP_S v8f32:$xj)>; +def : Pat<(fdiv vsplatf64_fpimm_eq_1, v4f64:$xj), + (XVFRECIP_D v4f64:$xj)>; + +// XVFRSQRT_{S/D} +def : Pat<(fdiv vsplatf32_fpimm_eq_1, (fsqrt v8f32:$xj)), + (XVFRSQRT_S v8f32:$xj)>; +def : Pat<(fdiv vsplatf64_fpimm_eq_1, (fsqrt v4f64:$xj)), + (XVFRSQRT_D v4f64:$xj)>; + +// XVSEQ[I]_{B/H/W/D} +defm : PatCCXrSimm5; +defm : PatCCXrXr; + +// XVSLE[I]_{B/H/W/D}[U] +defm : PatCCXrSimm5; +defm : PatCCXrUimm5; +defm : PatCCXrXr; +defm : PatCCXrXrU; + +// XVSLT[I]_{B/H/W/D}[U] +defm : PatCCXrSimm5; +defm : PatCCXrUimm5; +defm : PatCCXrXr; +defm : PatCCXrXrU; + +// XVFCMP.cond.{S/D} +defm : PatCCXrXrF; +defm : PatCCXrXrF; +defm : PatCCXrXrF; + +defm : PatCCXrXrF; +defm : PatCCXrXrF; +defm : PatCCXrXrF; + +defm : PatCCXrXrF; +defm : PatCCXrXrF; +defm : PatCCXrXrF; + +defm : PatCCXrXrF; +defm : PatCCXrXrF; +defm : PatCCXrXrF; + +defm : PatCCXrXrF; +defm : PatCCXrXrF; + // PseudoXVINSGR2VR_{B/H} def : Pat<(vector_insert v32i8:$xd, GRLenVT:$rj, uimm5:$imm), (PseudoXVINSGR2VR_B v32i8:$xd, GRLenVT:$rj, uimm5:$imm)>; @@ -1488,6 +1631,10 @@ foreach vt = [v32i8, v16i16, v8i32, v4i64, v8f32, v4f64] in def : Pat<(vt (vselect LASX256:$xa, LASX256:$xk, LASX256:$xj)), (XVBITSEL_V LASX256:$xj, LASX256:$xk, LASX256:$xa)>; +// fneg +def : Pat<(fneg (v8f32 LASX256:$xj)), (XVBITREVI_W LASX256:$xj, 31)>; +def : Pat<(fneg (v4f64 LASX256:$xj)), (XVBITREVI_D LASX256:$xj, 63)>; + } // Predicates = [HasExtLASX] /// Intrinsic pattern diff --git a/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td b/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td index 5800ff6f6266..e468176885d7 100644 --- a/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td +++ b/llvm/lib/Target/LoongArch/LoongArchLSXInstrInfo.td @@ -95,6 +95,29 @@ def vsplati64_imm_eq_63 : PatFrags<(ops), [(build_vector), Imm.getBitWidth() == EltTy.getSizeInBits() && Imm == 63; }]>; +def vsplatf32_fpimm_eq_1 + : PatFrags<(ops), [(bitconvert (v4i32 (build_vector))), + (bitconvert (v8i32 (build_vector)))], [{ + APInt Imm; + EVT EltTy = N->getValueType(0).getVectorElementType(); + N = N->getOperand(0).getNode(); + + return selectVSplat(N, Imm, EltTy.getSizeInBits()) && + Imm.getBitWidth() == EltTy.getSizeInBits() && + Imm == APFloat(+1.0f).bitcastToAPInt(); +}]>; +def vsplatf64_fpimm_eq_1 + : PatFrags<(ops), [(bitconvert (v2i64 (build_vector))), + (bitconvert (v4i64 (build_vector)))], [{ + APInt Imm; + EVT EltTy = N->getValueType(0).getVectorElementType(); + N = N->getOperand(0).getNode(); + + return selectVSplat(N, Imm, EltTy.getSizeInBits()) && + Imm.getBitWidth() == EltTy.getSizeInBits() && + Imm == APFloat(+1.0).bitcastToAPInt(); +}]>; + def vsplati8imm7 : PatFrag<(ops node:$reg), (and node:$reg, vsplati8_imm_eq_7)>; def vsplati16imm15 : PatFrag<(ops node:$reg), @@ -1173,6 +1196,13 @@ multiclass PatVr { (!cast(Inst#"_D") LSX128:$vj)>; } +multiclass PatVrF { + def : Pat<(v4f32 (OpNode (v4f32 LSX128:$vj))), + (!cast(Inst#"_S") LSX128:$vj)>; + def : Pat<(v2f64 (OpNode (v2f64 LSX128:$vj))), + (!cast(Inst#"_D") LSX128:$vj)>; +} + multiclass PatVrVr { def : Pat<(OpNode (v16i8 LSX128:$vj), (v16i8 LSX128:$vk)), (!cast(Inst#"_B") LSX128:$vj, LSX128:$vk)>; @@ -1261,6 +1291,65 @@ multiclass PatShiftVrUimm { (!cast(Inst#"_D") LSX128:$vj, uimm6:$imm)>; } +multiclass PatCCVrSimm5 { + def : Pat<(v16i8 (setcc (v16i8 LSX128:$vj), + (v16i8 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_B") LSX128:$vj, simm5:$imm)>; + def : Pat<(v8i16 (setcc (v8i16 LSX128:$vj), + (v8i16 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_H") LSX128:$vj, simm5:$imm)>; + def : Pat<(v4i32 (setcc (v4i32 LSX128:$vj), + (v4i32 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_W") LSX128:$vj, simm5:$imm)>; + def : Pat<(v2i64 (setcc (v2i64 LSX128:$vj), + (v2i64 (SplatPat_simm5 simm5:$imm)), CC)), + (!cast(Inst#"_D") LSX128:$vj, simm5:$imm)>; +} + +multiclass PatCCVrUimm5 { + def : Pat<(v16i8 (setcc (v16i8 LSX128:$vj), + (v16i8 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_BU") LSX128:$vj, uimm5:$imm)>; + def : Pat<(v8i16 (setcc (v8i16 LSX128:$vj), + (v8i16 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_HU") LSX128:$vj, uimm5:$imm)>; + def : Pat<(v4i32 (setcc (v4i32 LSX128:$vj), + (v4i32 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_WU") LSX128:$vj, uimm5:$imm)>; + def : Pat<(v2i64 (setcc (v2i64 LSX128:$vj), + (v2i64 (SplatPat_uimm5 uimm5:$imm)), CC)), + (!cast(Inst#"_DU") LSX128:$vj, uimm5:$imm)>; +} + +multiclass PatCCVrVr { + def : Pat<(v16i8 (setcc (v16i8 LSX128:$vj), (v16i8 LSX128:$vk), CC)), + (!cast(Inst#"_B") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v8i16 (setcc (v8i16 LSX128:$vj), (v8i16 LSX128:$vk), CC)), + (!cast(Inst#"_H") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v4i32 (setcc (v4i32 LSX128:$vj), (v4i32 LSX128:$vk), CC)), + (!cast(Inst#"_W") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v2i64 (setcc (v2i64 LSX128:$vj), (v2i64 LSX128:$vk), CC)), + (!cast(Inst#"_D") LSX128:$vj, LSX128:$vk)>; +} + +multiclass PatCCVrVrU { + def : Pat<(v16i8 (setcc (v16i8 LSX128:$vj), (v16i8 LSX128:$vk), CC)), + (!cast(Inst#"_BU") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v8i16 (setcc (v8i16 LSX128:$vj), (v8i16 LSX128:$vk), CC)), + (!cast(Inst#"_HU") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v4i32 (setcc (v4i32 LSX128:$vj), (v4i32 LSX128:$vk), CC)), + (!cast(Inst#"_WU") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v2i64 (setcc (v2i64 LSX128:$vj), (v2i64 LSX128:$vk), CC)), + (!cast(Inst#"_DU") LSX128:$vj, LSX128:$vk)>; +} + +multiclass PatCCVrVrF { + def : Pat<(v4i32 (setcc (v4f32 LSX128:$vj), (v4f32 LSX128:$vk), CC)), + (!cast(Inst#"_S") LSX128:$vj, LSX128:$vk)>; + def : Pat<(v2i64 (setcc (v2f64 LSX128:$vj), (v2f64 LSX128:$vk), CC)), + (!cast(Inst#"_D") LSX128:$vj, LSX128:$vk)>; +} + let Predicates = [HasExtLSX] in { // VADD_{B/H/W/D} @@ -1466,6 +1555,83 @@ def : Pat<(fma v4f32:$vj, v4f32:$vk, v4f32:$va), def : Pat<(fma v2f64:$vj, v2f64:$vk, v2f64:$va), (VFMADD_D v2f64:$vj, v2f64:$vk, v2f64:$va)>; +// VFMSUB_{S/D} +def : Pat<(fma v4f32:$vj, v4f32:$vk, (fneg v4f32:$va)), + (VFMSUB_S v4f32:$vj, v4f32:$vk, v4f32:$va)>; +def : Pat<(fma v2f64:$vj, v2f64:$vk, (fneg v2f64:$va)), + (VFMSUB_D v2f64:$vj, v2f64:$vk, v2f64:$va)>; + +// VFNMADD_{S/D} +def : Pat<(fneg (fma v4f32:$vj, v4f32:$vk, v4f32:$va)), + (VFNMADD_S v4f32:$vj, v4f32:$vk, v4f32:$va)>; +def : Pat<(fneg (fma v2f64:$vj, v2f64:$vk, v2f64:$va)), + (VFNMADD_D v2f64:$vj, v2f64:$vk, v2f64:$va)>; +def : Pat<(fma_nsz (fneg v4f32:$vj), v4f32:$vk, (fneg v4f32:$va)), + (VFNMADD_S v4f32:$vj, v4f32:$vk, v4f32:$va)>; +def : Pat<(fma_nsz (fneg v2f64:$vj), v2f64:$vk, (fneg v2f64:$va)), + (VFNMADD_D v2f64:$vj, v2f64:$vk, v2f64:$va)>; + +// VFNMSUB_{S/D} +def : Pat<(fneg (fma v4f32:$vj, v4f32:$vk, (fneg v4f32:$va))), + (VFNMSUB_S v4f32:$vj, v4f32:$vk, v4f32:$va)>; +def : Pat<(fneg (fma v2f64:$vj, v2f64:$vk, (fneg v2f64:$va))), + (VFNMSUB_D v2f64:$vj, v2f64:$vk, v2f64:$va)>; +def : Pat<(fma_nsz (fneg v4f32:$vj), v4f32:$vk, v4f32:$va), + (VFNMSUB_S v4f32:$vj, v4f32:$vk, v4f32:$va)>; +def : Pat<(fma_nsz (fneg v2f64:$vj), v2f64:$vk, v2f64:$va), + (VFNMSUB_D v2f64:$vj, v2f64:$vk, v2f64:$va)>; + +// VFSQRT_{S/D} +defm : PatVrF; + +// VFRECIP_{S/D} +def : Pat<(fdiv vsplatf32_fpimm_eq_1, v4f32:$vj), + (VFRECIP_S v4f32:$vj)>; +def : Pat<(fdiv vsplatf64_fpimm_eq_1, v2f64:$vj), + (VFRECIP_D v2f64:$vj)>; + +// VFRSQRT_{S/D} +def : Pat<(fdiv vsplatf32_fpimm_eq_1, (fsqrt v4f32:$vj)), + (VFRSQRT_S v4f32:$vj)>; +def : Pat<(fdiv vsplatf64_fpimm_eq_1, (fsqrt v2f64:$vj)), + (VFRSQRT_D v2f64:$vj)>; + +// VSEQ[I]_{B/H/W/D} +defm : PatCCVrSimm5; +defm : PatCCVrVr; + +// VSLE[I]_{B/H/W/D}[U] +defm : PatCCVrSimm5; +defm : PatCCVrUimm5; +defm : PatCCVrVr; +defm : PatCCVrVrU; + +// VSLT[I]_{B/H/W/D}[U] +defm : PatCCVrSimm5; +defm : PatCCVrUimm5; +defm : PatCCVrVr; +defm : PatCCVrVrU; + +// VFCMP.cond.{S/D} +defm : PatCCVrVrF; +defm : PatCCVrVrF; +defm : PatCCVrVrF; + +defm : PatCCVrVrF; +defm : PatCCVrVrF; +defm : PatCCVrVrF; + +defm : PatCCVrVrF; +defm : PatCCVrVrF; +defm : PatCCVrVrF; + +defm : PatCCVrVrF; +defm : PatCCVrVrF; +defm : PatCCVrVrF; + +defm : PatCCVrVrF; +defm : PatCCVrVrF; + // VINSGR2VR_{B/H/W/D} def : Pat<(vector_insert v16i8:$vd, GRLenVT:$rj, uimm4:$imm), (VINSGR2VR_B v16i8:$vd, GRLenVT:$rj, uimm4:$imm)>; @@ -1572,6 +1738,10 @@ foreach vt = [v16i8, v8i16, v4i32, v2i64, v4f32, v2f64] in def : Pat<(vt (vselect LSX128:$va, LSX128:$vk, LSX128:$vj)), (VBITSEL_V LSX128:$vj, LSX128:$vk, LSX128:$va)>; +// fneg +def : Pat<(fneg (v4f32 LSX128:$vj)), (VBITREVI_W LSX128:$vj, 31)>; +def : Pat<(fneg (v2f64 LSX128:$vj)), (VBITREVI_D LSX128:$vj, 63)>; + } // Predicates = [HasExtLSX] /// Intrinsic pattern diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp index a35916d2ad21..14bcef7c7d26 100644 --- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp +++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.cpp @@ -162,12 +162,13 @@ void LoongArchAsmBackend::applyFixup(const MCAssembler &Asm, bool LoongArchAsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { if (Fixup.getKind() >= FirstLiteralRelocationKind) return true; switch (Fixup.getTargetKind()) { default: - return STI.hasFeature(LoongArch::FeatureRelax); + return STI->hasFeature(LoongArch::FeatureRelax); case FK_Data_1: case FK_Data_2: case FK_Data_4: diff --git a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h index f840f9fa2b6a..d1fbf788e8a8 100644 --- a/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h +++ b/llvm/lib/Target/LoongArch/MCTargetDesc/LoongArchAsmBackend.h @@ -41,7 +41,8 @@ public: const MCSubtargetInfo *STI) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value, const MCRelaxableFragment *DF, diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp index 7eca49e709a0..fc95b61fd4df 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.cpp @@ -544,7 +544,8 @@ bool MipsAsmBackend::writeNopData(raw_ostream &OS, uint64_t Count, bool MipsAsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { if (Fixup.getKind() >= FirstLiteralRelocationKind) return true; const unsigned FixupKind = Fixup.getKind(); diff --git a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h index 228a0b4c407c..2dd68b601238 100644 --- a/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h +++ b/llvm/lib/Target/Mips/MCTargetDesc/MipsAsmBackend.h @@ -68,7 +68,8 @@ public: const MCSubtargetInfo *STI) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; bool isMicroMips(const MCSymbol *Sym) const override; }; // class MipsAsmBackend diff --git a/llvm/lib/Target/Mips/Mips16FrameLowering.cpp b/llvm/lib/Target/Mips/Mips16FrameLowering.cpp index 4f4e3f3f2ed7..10c953bb344a 100644 --- a/llvm/lib/Target/Mips/Mips16FrameLowering.cpp +++ b/llvm/lib/Target/Mips/Mips16FrameLowering.cpp @@ -30,7 +30,6 @@ #include "llvm/MC/MachineLocation.h" #include "llvm/Support/MathExtras.h" #include "llvm/CodeGen/TargetFrameLowering.h" -#include #include #include diff --git a/llvm/lib/Target/Mips/MipsISelLowering.h b/llvm/lib/Target/Mips/MipsISelLowering.h index 81d37ff2f065..c17e51f44651 100644 --- a/llvm/lib/Target/Mips/MipsISelLowering.h +++ b/llvm/lib/Target/Mips/MipsISelLowering.h @@ -31,7 +31,6 @@ #include "llvm/IR/Type.h" #include "llvm/Target/TargetMachine.h" #include -#include #include #include #include diff --git a/llvm/lib/Target/Mips/MipsMCInstLower.cpp b/llvm/lib/Target/Mips/MipsMCInstLower.cpp index 7b58cb90ab87..b0642f3d1ff2 100644 --- a/llvm/lib/Target/Mips/MipsMCInstLower.cpp +++ b/llvm/lib/Target/Mips/MipsMCInstLower.cpp @@ -21,7 +21,6 @@ #include "llvm/MC/MCExpr.h" #include "llvm/MC/MCInst.h" #include "llvm/Support/ErrorHandling.h" -#include using namespace llvm; diff --git a/llvm/lib/Target/Mips/MipsSEISelLowering.cpp b/llvm/lib/Target/Mips/MipsSEISelLowering.cpp index b84f304373f6..5c34067c8888 100644 --- a/llvm/lib/Target/Mips/MipsSEISelLowering.cpp +++ b/llvm/lib/Target/Mips/MipsSEISelLowering.cpp @@ -15,7 +15,6 @@ #include "MipsRegisterInfo.h" #include "MipsSubtarget.h" #include "llvm/ADT/APInt.h" -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/CallingConvLower.h" diff --git a/llvm/lib/Target/NVPTX/NVPTX.td b/llvm/lib/Target/NVPTX/NVPTX.td index 02fa2a4ee81e..f2a4ce381b40 100644 --- a/llvm/lib/Target/NVPTX/NVPTX.td +++ b/llvm/lib/Target/NVPTX/NVPTX.td @@ -24,23 +24,24 @@ include "NVPTXInstrInfo.td" // TableGen in NVPTXGenSubtarget.inc. //===----------------------------------------------------------------------===// -class FeatureSM: - SubtargetFeature<"sm_"# version, "SmVersion", - "" # version, - "Target SM " # version>; -def SM90a: FeatureSM<90>; +class FeatureSM: + SubtargetFeature<"sm_"# sm, "FullSmVersion", + "" # value, + "Target SM " # sm>; class FeaturePTX: SubtargetFeature<"ptx"# version, "PTXVersion", "" # version, "Use PTX version " # version>; -foreach version = [20, 21, 30, 32, 35, 37, 50, 52, 53, - 60, 61, 62, 70, 72, 75, 80, 86, 87, 89, 90] in - def SM#version: FeatureSM; +foreach sm = [20, 21, 30, 32, 35, 37, 50, 52, 53, + 60, 61, 62, 70, 72, 75, 80, 86, 87, 89, 90] in + def SM#sm: FeatureSM<""#sm, !mul(sm, 10)>; + +def SM90a: FeatureSM<"90a", 901>; foreach version = [32, 40, 41, 42, 43, 50, 60, 61, 63, 64, 65, - 70, 71, 72, 73, 74, 75, 76, 77, 78, 80, 81] in + 70, 71, 72, 73, 74, 75, 76, 77, 78, 80, 81, 82, 83] in def PTX#version: FeaturePTX; //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp index 61285c6ba98d..f5d8abaf847a 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -766,6 +766,17 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM, AddPromotedToType(Op, MVT::bf16, MVT::f32); } + // sm_80 only has conversions between f32 and bf16. Custom lower all other + // bf16 conversions. + if (STI.hasBF16Math() && + (STI.getSmVersion() < 90 || STI.getPTXVersion() < 78)) { + for (MVT VT : {MVT::i1, MVT::i16, MVT::i32, MVT::i64}) { + setOperationAction( + {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT, ISD::FP_TO_UINT}, + VT, Custom); + } + } + setOperationAction(ISD::FROUND, MVT::f16, Promote); setOperationAction(ISD::FROUND, MVT::v2f16, Expand); setOperationAction(ISD::FROUND, MVT::v2bf16, Expand); @@ -2580,6 +2591,37 @@ SDValue NVPTXTargetLowering::LowerFROUND64(SDValue Op, return DAG.getNode(ISD::SELECT, SL, VT, IsLarge, A, RoundedA); } +SDValue NVPTXTargetLowering::LowerINT_TO_FP(SDValue Op, + SelectionDAG &DAG) const { + assert(STI.getSmVersion() < 90 || STI.getPTXVersion() < 78); + + if (Op.getValueType() == MVT::bf16) { + SDLoc Loc(Op); + return DAG.getNode( + ISD::FP_ROUND, Loc, MVT::bf16, + DAG.getNode(Op.getOpcode(), Loc, MVT::f32, Op.getOperand(0)), + DAG.getIntPtrConstant(0, Loc)); + } + + // Everything else is considered legal. + return Op; +} + +SDValue NVPTXTargetLowering::LowerFP_TO_INT(SDValue Op, + SelectionDAG &DAG) const { + assert(STI.getSmVersion() < 90 || STI.getPTXVersion() < 78); + + if (Op.getOperand(0).getValueType() == MVT::bf16) { + SDLoc Loc(Op); + return DAG.getNode( + Op.getOpcode(), Loc, Op.getValueType(), + DAG.getNode(ISD::FP_EXTEND, Loc, MVT::f32, Op.getOperand(0))); + } + + // Everything else is considered legal. + return Op; +} + static SDValue LowerVectorArith(SDValue Op, SelectionDAG &DAG) { SDLoc DL(Op); if (Op.getValueType() != MVT::v2i16) @@ -2636,6 +2678,12 @@ NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const { return LowerSelect(Op, DAG); case ISD::FROUND: return LowerFROUND(Op, DAG); + case ISD::SINT_TO_FP: + case ISD::UINT_TO_FP: + return LowerINT_TO_FP(Op, DAG); + case ISD::FP_TO_SINT: + case ISD::FP_TO_UINT: + return LowerFP_TO_INT(Op, DAG); case ISD::VAARG: return LowerVAARG(Op, DAG); case ISD::VASTART: diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h index 54e34dedc667..cd6bcb048c5f 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h @@ -607,6 +607,9 @@ private: SDValue LowerFROUND32(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFROUND64(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerINT_TO_FP(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const; SDValue LowerLOADi1(SDValue Op, SelectionDAG &DAG) const; diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td index 58ff193f2437..13665985f52e 100644 --- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td +++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td @@ -561,7 +561,7 @@ multiclass F2_Support_Half { [(set Int16Regs:$dst, (OpNode (bf16 Int16Regs:$a)))]>, Requires<[hasSM<80>, hasPTX<70>]>; def bf16x2 : NVPTXInst<(outs Int32Regs:$dst), (ins Int32Regs:$a), - !strconcat(OpcStr, ".v2bf16 \t$dst, $a;"), + !strconcat(OpcStr, ".bf16x2 \t$dst, $a;"), [(set Int32Regs:$dst, (OpNode (v2bf16 Int32Regs:$a)))]>, Requires<[hasSM<80>, hasPTX<70>]>; def f16_ftz : NVPTXInst<(outs Int16Regs:$dst), (ins Int16Regs:$a), diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp b/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp index 7fa64af196b9..420065585b38 100644 --- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.cpp @@ -36,6 +36,11 @@ NVPTXSubtarget &NVPTXSubtarget::initializeSubtargetDependencies(StringRef CPU, ParseSubtargetFeatures(TargetName, /*TuneCPU*/ TargetName, FS); + // Re-map SM version numbers, SmVersion carries the regular SMs which do + // have relative order, while FullSmVersion allows distinguishing sm_90 from + // sm_90a, which would *not* be a subset of sm_91. + SmVersion = getSmVersion(); + // Set default to PTX 6.0 (CUDA 9.0) if (PTXVersion == 0) { PTXVersion = 60; @@ -48,7 +53,7 @@ NVPTXSubtarget::NVPTXSubtarget(const Triple &TT, const std::string &CPU, const std::string &FS, const NVPTXTargetMachine &TM) : NVPTXGenSubtargetInfo(TT, CPU, /*TuneCPU*/ CPU, FS), PTXVersion(0), - SmVersion(20), TM(TM), + FullSmVersion(200), SmVersion(getSmVersion()), TM(TM), TLInfo(TM, initializeSubtargetDependencies(CPU, FS)) {} bool NVPTXSubtarget::hasImageHandles() const { diff --git a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h index 93af11c258b4..3ca4c1a24c79 100644 --- a/llvm/lib/Target/NVPTX/NVPTXSubtarget.h +++ b/llvm/lib/Target/NVPTX/NVPTXSubtarget.h @@ -35,7 +35,12 @@ class NVPTXSubtarget : public NVPTXGenSubtargetInfo { // PTX version x.y is represented as 10*x+y, e.g. 3.1 == 31 unsigned PTXVersion; - // SM version x.y is represented as 10*x+y, e.g. 3.1 == 31 + // Full SM version x.y is represented as 100*x+10*y+feature, e.g. 3.1 == 310 + // sm_90a == 901 + unsigned int FullSmVersion; + + // SM version x.y is represented as 10*x+y, e.g. 3.1 == 31. Derived from + // FullSmVersion. unsigned int SmVersion; const NVPTXTargetMachine &TM; @@ -80,7 +85,15 @@ public: bool allowFP16Math() const; bool hasMaskOperator() const { return PTXVersion >= 71; } bool hasNoReturn() const { return SmVersion >= 30 && PTXVersion >= 64; } - unsigned int getSmVersion() const { return SmVersion; } + unsigned int getFullSmVersion() const { return FullSmVersion; } + unsigned int getSmVersion() const { return getFullSmVersion() / 10; } + // GPUs with "a" suffix have include architecture-accelerated features that + // are supported on the specified architecture only, hence such targets do not + // follow the onion layer model. hasAAFeatures() allows distinguishing such + // GPU variants from the base GPU architecture. + // - 0 represents base GPU model, + // - non-zero value identifies particular architecture-accelerated variant. + bool hasAAFeatures() const { return getFullSmVersion() % 10; } std::string getTargetName() const { return TargetName; } // Get maximum value of required alignments among the supported data types. diff --git a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp index ed52b9a9cf27..a33f44542eb5 100644 --- a/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp +++ b/llvm/lib/Target/PowerPC/AsmParser/PPCAsmParser.cpp @@ -886,6 +886,26 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst, Inst = TmpInst; break; } + case PPC::PLA8: + case PPC::PLA: { + MCInst TmpInst; + TmpInst.setOpcode(Opcode == PPC::PLA ? PPC::PADDI : PPC::PADDI8); + TmpInst.addOperand(Inst.getOperand(0)); + TmpInst.addOperand(Inst.getOperand(1)); + TmpInst.addOperand(Inst.getOperand(2)); + Inst = TmpInst; + break; + } + case PPC::PLA8pc: + case PPC::PLApc: { + MCInst TmpInst; + TmpInst.setOpcode(Opcode == PPC::PLApc ? PPC::PADDIpc : PPC::PADDI8pc); + TmpInst.addOperand(Inst.getOperand(0)); + TmpInst.addOperand(MCOperand::createImm(0)); + TmpInst.addOperand(Inst.getOperand(1)); + Inst = TmpInst; + break; + } case PPC::SUBI: { MCInst TmpInst; TmpInst.setOpcode(PPC::ADDI); @@ -895,6 +915,15 @@ void PPCAsmParser::ProcessInstruction(MCInst &Inst, Inst = TmpInst; break; } + case PPC::PSUBI: { + MCInst TmpInst; + TmpInst.setOpcode(PPC::PADDI); + TmpInst.addOperand(Inst.getOperand(0)); + TmpInst.addOperand(Inst.getOperand(1)); + addNegOperand(TmpInst, Inst.getOperand(2), getContext()); + Inst = TmpInst; + break; + } case PPC::SUBIS: { MCInst TmpInst; TmpInst.setOpcode(PPC::ADDIS); diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp index 8bd27571a750..251737ed1275 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCAsmBackend.cpp @@ -162,7 +162,8 @@ public: } bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override { + const MCValue &Target, + const MCSubtargetInfo *STI) override { MCFixupKind Kind = Fixup.getKind(); switch ((unsigned)Kind) { default: diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.h b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.h index 7d786ac13bb9..10204b184a49 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.h +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCELFStreamer.h @@ -13,7 +13,6 @@ #ifndef LLVM_LIB_TARGET_PPC_MCELFSTREAMER_PPCELFSTREAMER_H #define LLVM_LIB_TARGET_PPC_MCELFSTREAMER_PPCELFSTREAMER_H -#include "llvm/ADT/SmallVector.h" #include "llvm/MC/MCELFStreamer.h" #include diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp index a5dc0b45b13c..4716e37b3443 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCMCAsmInfo.cpp @@ -12,7 +12,6 @@ #include "PPCMCAsmInfo.h" #include "llvm/TargetParser/Triple.h" -#include using namespace llvm; diff --git a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp index 284e52c298a2..80c37f82bf29 100644 --- a/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp +++ b/llvm/lib/Target/PowerPC/MCTargetDesc/PPCPredicates.cpp @@ -12,7 +12,6 @@ #include "PPCPredicates.h" #include "llvm/Support/ErrorHandling.h" -#include using namespace llvm; PPC::Predicate PPC::InvertPredicate(PPC::Predicate Opcode) { diff --git a/llvm/lib/Target/PowerPC/P10InstrResources.td b/llvm/lib/Target/PowerPC/P10InstrResources.td index 66a050955631..3bbc5a63ca7a 100644 --- a/llvm/lib/Target/PowerPC/P10InstrResources.td +++ b/llvm/lib/Target/PowerPC/P10InstrResources.td @@ -875,7 +875,7 @@ def : InstRW<[P10W_FX_3C, P10W_DISP_ANY], MCRXRX, MFCTR, MFCTR8, MFLR, MFLR8, - WAIT + WAIT, WAITP10 )>; // 3 Cycles ALU operations, 1 input operands @@ -1884,6 +1884,7 @@ def : InstRW<[P10W_ST_3C, P10W_DISP_EVEN, P10W_DISP_ANY, P10ST_Read, P10ST_Read, def : InstRW<[P10W_ST_3C, P10W_DISP_EVEN, P10W_FX_3C, P10W_DISP_ANY], (instrs ISYNC, + SYNCP10, SYNC )>; diff --git a/llvm/lib/Target/PowerPC/PPC.h b/llvm/lib/Target/PowerPC/PPC.h index 086b61a93a43..3d9ea5608193 100644 --- a/llvm/lib/Target/PowerPC/PPC.h +++ b/llvm/lib/Target/PowerPC/PPC.h @@ -102,79 +102,99 @@ class ModulePass; // PPC Specific MachineOperand flags. MO_NO_FLAG, + /// On PPC, the 12 bits are not enough for all target operand flags. + /// Treat all PPC target flags as direct flags. To define new flag that is + /// combination of other flags, add new enum entry instead of combining + /// existing flags. See example MO_GOT_TPREL_PCREL_FLAG. + /// On a symbol operand "FOO", this indicates that the reference is actually /// to "FOO@plt". This is used for calls and jumps to external functions /// and for PIC calls on 32-bit ELF systems. - MO_PLT = 1, + MO_PLT, /// MO_PIC_FLAG - If this bit is set, the symbol reference is relative to /// the function's picbase, e.g. lo16(symbol-picbase). - MO_PIC_FLAG = 2, + MO_PIC_FLAG, /// MO_PCREL_FLAG - If this bit is set, the symbol reference is relative to /// the current instruction address(pc), e.g., var@pcrel. Fixup is VK_PCREL. - MO_PCREL_FLAG = 4, + MO_PCREL_FLAG, /// MO_GOT_FLAG - If this bit is set the symbol reference is to be computed /// via the GOT. For example when combined with the MO_PCREL_FLAG it should /// produce the relocation @got@pcrel. Fixup is VK_PPC_GOT_PCREL. - MO_GOT_FLAG = 8, + MO_GOT_FLAG, - // MO_PCREL_OPT_FLAG - If this bit is set the operand is part of a - // PC Relative linker optimization. - MO_PCREL_OPT_FLAG = 16, + /// MO_PCREL_OPT_FLAG - If this bit is set the operand is part of a + /// PC Relative linker optimization. + MO_PCREL_OPT_FLAG, /// MO_TLSGD_FLAG - If this bit is set the symbol reference is relative to /// TLS General Dynamic model for Linux and the variable offset of TLS /// General Dynamic model for AIX. - MO_TLSGD_FLAG = 32, + MO_TLSGD_FLAG, /// MO_TPREL_FLAG - If this bit is set, the symbol reference is relative to /// the thread pointer and the symbol can be used for the TLS Initial Exec /// and Local Exec models. - MO_TPREL_FLAG = 64, + MO_TPREL_FLAG, /// MO_TLSLD_FLAG - If this bit is set the symbol reference is relative to /// TLS Local Dynamic model. - MO_TLSLD_FLAG = 128, + MO_TLSLD_FLAG, /// MO_TLSGDM_FLAG - If this bit is set the symbol reference is relative /// to the region handle of TLS General Dynamic model for AIX. - MO_TLSGDM_FLAG = 256, + MO_TLSGDM_FLAG, /// MO_GOT_TLSGD_PCREL_FLAG - A combintaion of flags, if these bits are set /// they should produce the relocation @got@tlsgd@pcrel. /// Fix up is VK_PPC_GOT_TLSGD_PCREL - MO_GOT_TLSGD_PCREL_FLAG = MO_PCREL_FLAG | MO_GOT_FLAG | MO_TLSGD_FLAG, + /// MO_GOT_TLSGD_PCREL_FLAG = MO_PCREL_FLAG | MO_GOT_FLAG | MO_TLSGD_FLAG, + MO_GOT_TLSGD_PCREL_FLAG, /// MO_GOT_TLSLD_PCREL_FLAG - A combintaion of flags, if these bits are set /// they should produce the relocation @got@tlsld@pcrel. /// Fix up is VK_PPC_GOT_TLSLD_PCREL - MO_GOT_TLSLD_PCREL_FLAG = MO_PCREL_FLAG | MO_GOT_FLAG | MO_TLSLD_FLAG, + /// MO_GOT_TLSLD_PCREL_FLAG = MO_PCREL_FLAG | MO_GOT_FLAG | MO_TLSLD_FLAG, + MO_GOT_TLSLD_PCREL_FLAG, /// MO_GOT_TPREL_PCREL_FLAG - A combintaion of flags, if these bits are set /// they should produce the relocation @got@tprel@pcrel. /// Fix up is VK_PPC_GOT_TPREL_PCREL - MO_GOT_TPREL_PCREL_FLAG = MO_GOT_FLAG | MO_TPREL_FLAG | MO_PCREL_FLAG, - - /// The next are not flags but distinct values. - MO_ACCESS_MASK = 0xf00, + /// MO_GOT_TPREL_PCREL_FLAG = MO_GOT_FLAG | MO_TPREL_FLAG | MO_PCREL_FLAG, + MO_GOT_TPREL_PCREL_FLAG, /// MO_LO, MO_HA - lo16(symbol) and ha16(symbol) - MO_LO = 1 << 8, - MO_HA = 2 << 8, + MO_LO, + MO_HA, - MO_TPREL_LO = 4 << 8, - MO_TPREL_HA = 3 << 8, + MO_TPREL_LO, + MO_TPREL_HA, /// These values identify relocations on immediates folded /// into memory operations. - MO_DTPREL_LO = 5 << 8, - MO_TLSLD_LO = 6 << 8, - MO_TOC_LO = 7 << 8, + MO_DTPREL_LO, + MO_TLSLD_LO, + MO_TOC_LO, + + /// Symbol for VK_PPC_TLS fixup attached to an ADD instruction + MO_TLS, + + /// MO_PIC_HA_FLAG = MO_PIC_FLAG | MO_HA + MO_PIC_HA_FLAG, + + /// MO_PIC_LO_FLAG = MO_PIC_FLAG | MO_LO + MO_PIC_LO_FLAG, + + /// MO_TPREL_PCREL_FLAG = MO_PCREL_FLAG | MO_TPREL_FLAG + MO_TPREL_PCREL_FLAG, + + /// MO_TPREL_PCREL_FLAG = MO_PCREL_FLAG | MO_TLS + MO_TLS_PCREL_FLAG, - // Symbol for VK_PPC_TLS fixup attached to an ADD instruction - MO_TLS = 8 << 8 + /// MO_GOT_PCREL_FLAG = MO_PCREL_FLAG | MO_GOT_FLAG + MO_GOT_PCREL_FLAG, }; } // end namespace PPCII diff --git a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp index 4f15ba497d84..4b551bc51c4f 100644 --- a/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp +++ b/llvm/lib/Target/PowerPC/PPCAsmPrinter.cpp @@ -715,25 +715,11 @@ static MCSymbol *getMCSymbolForTOCPseudoMO(const MachineOperand &MO, } } -static bool hasTLSFlag(const MachineOperand &MO) { - unsigned Flags = MO.getTargetFlags(); - if (Flags & PPCII::MO_TLSGD_FLAG || Flags & PPCII::MO_TPREL_FLAG || - Flags & PPCII::MO_TLSLD_FLAG || Flags & PPCII::MO_TLSGDM_FLAG) - return true; - - if (Flags == PPCII::MO_TPREL_LO || Flags == PPCII::MO_TPREL_HA || - Flags == PPCII::MO_DTPREL_LO || Flags == PPCII::MO_TLSLD_LO || - Flags == PPCII::MO_TLS) - return true; - - return false; -} - static PPCAsmPrinter::TOCEntryType getTOCEntryTypeForMO(const MachineOperand &MO) { // Use the target flags to determine if this MO is Thread Local. // If we don't do this it comes out as Global. - if (hasTLSFlag(MO)) + if (PPCInstrInfo::hasTLSFlag(MO.getTargetFlags())) return PPCAsmPrinter::TOCType_ThreadLocal; switch (MO.getType()) { @@ -830,7 +816,10 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { // For TLS initial-exec and local-exec accesses on AIX, we have one TOC // entry for the symbol (with the variable offset), which is differentiated // by MO_TPREL_FLAG. - if (MO.getTargetFlags() & PPCII::MO_TPREL_FLAG) { + unsigned Flag = MO.getTargetFlags(); + if (Flag == PPCII::MO_TPREL_FLAG || + Flag == PPCII::MO_GOT_TPREL_PCREL_FLAG || + Flag == PPCII::MO_TPREL_PCREL_FLAG) { assert(MO.isGlobal() && "Only expecting a global MachineOperand here!\n"); TLSModel::Model Model = TM.getTLSModel(MO.getGlobal()); if (Model == TLSModel::LocalExec) @@ -842,9 +831,9 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { // For GD TLS access on AIX, we have two TOC entries for the symbol (one for // the variable offset and the other for the region handle). They are // differentiated by MO_TLSGD_FLAG and MO_TLSGDM_FLAG. - if (MO.getTargetFlags() & PPCII::MO_TLSGDM_FLAG) + if (Flag == PPCII::MO_TLSGDM_FLAG) return MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSGDM; - if (MO.getTargetFlags() & PPCII::MO_TLSGD_FLAG) + if (Flag == PPCII::MO_TLSGD_FLAG || Flag == PPCII::MO_GOT_TLSGD_PCREL_FLAG) return MCSymbolRefExpr::VariantKind::VK_PPC_AIX_TLSGD; return MCSymbolRefExpr::VariantKind::VK_None; }; @@ -1538,8 +1527,10 @@ void PPCAsmPrinter::emitInstruction(const MachineInstr *MI) { // The faster non-TOC-based local-exec sequence is represented by `addi` // with an immediate operand having the MO_TPREL_FLAG. Such an instruction // does not otherwise arise. - const MachineOperand &MO = MI->getOperand(2); - if ((MO.getTargetFlags() & PPCII::MO_TPREL_FLAG) != 0) { + unsigned Flag = MI->getOperand(2).getTargetFlags(); + if (Flag == PPCII::MO_TPREL_FLAG || + Flag == PPCII::MO_GOT_TPREL_PCREL_FLAG || + Flag == PPCII::MO_TPREL_PCREL_FLAG) { assert( Subtarget->hasAIXSmallLocalExecTLS() && "addi with thread-pointer only expected with local-exec small TLS"); diff --git a/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def b/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def index 5d97d187b296..8bbe315a2bb9 100644 --- a/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def +++ b/llvm/lib/Target/PowerPC/PPCBack2BackFusion.def @@ -958,6 +958,7 @@ FUSION_FEATURE(GeneralBack2Back, hasBack2BackFusion, -1, V_SET0B, V_SET0H, WAIT, + WAITP10, XOR, XOR8, XOR8_rec, diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index ca65048e2355..acaf98b62fd9 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -2972,7 +2972,7 @@ bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base, template static bool isValidPCRelNode(SDValue N) { Ty *PCRelCand = dyn_cast(N); - return PCRelCand && (PCRelCand->getTargetFlags() & PPCII::MO_PCREL_FLAG); + return PCRelCand && (PPCInstrInfo::hasPCRelFlag(PCRelCand->getTargetFlags())); } /// Returns true if this address is a PC Relative address. @@ -3133,8 +3133,8 @@ static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget, // Don't use the pic base if not in PIC relocation model. if (IsPIC) { - HiOpFlags |= PPCII::MO_PIC_FLAG; - LoOpFlags |= PPCII::MO_PIC_FLAG; + HiOpFlags = PPCII::MO_PIC_HA_FLAG; + LoOpFlags = PPCII::MO_PIC_LO_FLAG; } } @@ -3453,8 +3453,8 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddressLinux(SDValue Op, if (Model == TLSModel::LocalExec) { if (Subtarget.isUsingPCRelativeCalls()) { SDValue TLSReg = DAG.getRegister(PPC::X13, MVT::i64); - SDValue TGA = DAG.getTargetGlobalAddress( - GV, dl, PtrVT, 0, (PPCII::MO_PCREL_FLAG | PPCII::MO_TPREL_FLAG)); + SDValue TGA = DAG.getTargetGlobalAddress(GV, dl, PtrVT, 0, + PPCII::MO_TPREL_PCREL_FLAG); SDValue MatAddr = DAG.getNode(PPCISD::TLS_LOCAL_EXEC_MAT_ADDR, dl, PtrVT, TGA); return DAG.getNode(PPCISD::ADD_TLS, dl, PtrVT, TLSReg, MatAddr); @@ -3476,8 +3476,7 @@ SDValue PPCTargetLowering::LowerGlobalTLSAddressLinux(SDValue Op, SDValue TGA = DAG.getTargetGlobalAddress( GV, dl, PtrVT, 0, IsPCRel ? PPCII::MO_GOT_TPREL_PCREL_FLAG : 0); SDValue TGATLS = DAG.getTargetGlobalAddress( - GV, dl, PtrVT, 0, - IsPCRel ? (PPCII::MO_TLS | PPCII::MO_PCREL_FLAG) : PPCII::MO_TLS); + GV, dl, PtrVT, 0, IsPCRel ? PPCII::MO_TLS_PCREL_FLAG : PPCII::MO_TLS); SDValue TPOffset; if (IsPCRel) { SDValue MatPCRel = DAG.getNode(PPCISD::MAT_PCREL_ADDR, dl, PtrVT, TGA); @@ -3573,8 +3572,7 @@ SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op, EVT Ty = getPointerTy(DAG.getDataLayout()); if (isAccessedAsGotIndirect(Op)) { SDValue GA = DAG.getTargetGlobalAddress(GV, DL, Ty, GSDN->getOffset(), - PPCII::MO_PCREL_FLAG | - PPCII::MO_GOT_FLAG); + PPCII::MO_GOT_PCREL_FLAG); SDValue MatPCRel = DAG.getNode(PPCISD::MAT_PCREL_ADDR, DL, Ty, GA); SDValue Load = DAG.getLoad(MVT::i64, DL, DAG.getEntryNode(), MatPCRel, MachinePointerInfo()); diff --git a/llvm/lib/Target/PowerPC/PPCInstrFormats.td b/llvm/lib/Target/PowerPC/PPCInstrFormats.td index 6501c17dd810..5389f42a325c 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrFormats.td +++ b/llvm/lib/Target/PowerPC/PPCInstrFormats.td @@ -725,6 +725,38 @@ class XForm_24_sync opcode, bits<10> xo, dag OOL, dag IOL, let Inst{31} = 0; } +class XForm_IMM2_IMM2 opcode, bits<10> xo, dag OOL, dag IOL, + string asmstr, InstrItinClass itin, list pattern> + : I { + bits<2> L; + bits<2> PL; + + let Pattern = pattern; + let Inst{6-8} = 0; + let Inst{9-10} = L; + let Inst{11-13} = 0; + let Inst{14-15} = PL; + let Inst{16-20} = 0; + let Inst{21-30} = xo; + let Inst{31} = 0; +} + +class XForm_IMM3_IMM2 opcode, bits<10> xo, dag OOL, dag IOL, + string asmstr, InstrItinClass itin, list pattern> + : I { + bits<3> L; + bits<2> SC; + + let Pattern = pattern; + let Inst{6-7} = 0; + let Inst{8-10} = L; + let Inst{11-13} = 0; + let Inst{14-15} = SC; + let Inst{16-20} = 0; + let Inst{21-30} = xo; + let Inst{31} = 0; +} + class XForm_24_eieio opcode, bits<10> xo, dag OOL, dag IOL, string asmstr, InstrItinClass itin, list pattern> : XForm_24_sync { diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp index 21cd27d62711..d0a6cced1b19 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp @@ -2956,27 +2956,12 @@ unsigned PPCInstrInfo::getInstSizeInBytes(const MachineInstr &MI) const { std::pair PPCInstrInfo::decomposeMachineOperandsTargetFlags(unsigned TF) const { - const unsigned Mask = PPCII::MO_ACCESS_MASK; - return std::make_pair(TF & Mask, TF & ~Mask); + // PPC always uses a direct mask. + return std::make_pair(TF, 0u); } ArrayRef> PPCInstrInfo::getSerializableDirectMachineOperandTargetFlags() const { - using namespace PPCII; - static const std::pair TargetFlags[] = { - {MO_LO, "ppc-lo"}, - {MO_HA, "ppc-ha"}, - {MO_TPREL_LO, "ppc-tprel-lo"}, - {MO_TPREL_HA, "ppc-tprel-ha"}, - {MO_DTPREL_LO, "ppc-dtprel-lo"}, - {MO_TLSLD_LO, "ppc-tlsld-lo"}, - {MO_TOC_LO, "ppc-toc-lo"}, - {MO_TLS, "ppc-tls"}}; - return ArrayRef(TargetFlags); -} - -ArrayRef> -PPCInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const { using namespace PPCII; static const std::pair TargetFlags[] = { {MO_PLT, "ppc-plt"}, @@ -2985,12 +2970,26 @@ PPCInstrInfo::getSerializableBitmaskMachineOperandTargetFlags() const { {MO_GOT_FLAG, "ppc-got"}, {MO_PCREL_OPT_FLAG, "ppc-opt-pcrel"}, {MO_TLSGD_FLAG, "ppc-tlsgd"}, - {MO_TLSLD_FLAG, "ppc-tlsld"}, {MO_TPREL_FLAG, "ppc-tprel"}, + {MO_TLSLD_FLAG, "ppc-tlsld"}, {MO_TLSGDM_FLAG, "ppc-tlsgdm"}, {MO_GOT_TLSGD_PCREL_FLAG, "ppc-got-tlsgd-pcrel"}, {MO_GOT_TLSLD_PCREL_FLAG, "ppc-got-tlsld-pcrel"}, - {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"}}; + {MO_GOT_TPREL_PCREL_FLAG, "ppc-got-tprel-pcrel"}, + {MO_LO, "ppc-lo"}, + {MO_HA, "ppc-ha"}, + {MO_TPREL_LO, "ppc-tprel-lo"}, + {MO_TPREL_HA, "ppc-tprel-ha"}, + {MO_DTPREL_LO, "ppc-dtprel-lo"}, + {MO_TLSLD_LO, "ppc-tlsld-lo"}, + {MO_TOC_LO, "ppc-toc-lo"}, + {MO_TLS, "ppc-tls"}, + {MO_PIC_HA_FLAG, "ppc-ha-pic"}, + {MO_PIC_LO_FLAG, "ppc-lo-pic"}, + {MO_TPREL_PCREL_FLAG, "ppc-tprel-pcrel"}, + {MO_TLS_PCREL_FLAG, "ppc-tls-pcrel"}, + {MO_GOT_PCREL_FLAG, "ppc-got-pcrel"}, + }; return ArrayRef(TargetFlags); } diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h index 2f0b9498411a..75f9cd1c206d 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h @@ -14,6 +14,7 @@ #define LLVM_LIB_TARGET_POWERPC_PPCINSTRINFO_H #include "MCTargetDesc/PPCMCTargetDesc.h" +#include "PPC.h" #include "PPCRegisterInfo.h" #include "llvm/ADT/SmallSet.h" #include "llvm/CodeGen/TargetInstrInfo.h" @@ -283,6 +284,32 @@ public: return false; } + static bool hasPCRelFlag(unsigned TF) { + return TF == PPCII::MO_PCREL_FLAG || TF == PPCII::MO_GOT_TLSGD_PCREL_FLAG || + TF == PPCII::MO_GOT_TLSLD_PCREL_FLAG || + TF == PPCII::MO_GOT_TPREL_PCREL_FLAG || + TF == PPCII::MO_TPREL_PCREL_FLAG || TF == PPCII::MO_TLS_PCREL_FLAG || + TF == PPCII::MO_GOT_PCREL_FLAG; + } + + static bool hasGOTFlag(unsigned TF) { + return TF == PPCII::MO_GOT_FLAG || TF == PPCII::MO_GOT_TLSGD_PCREL_FLAG || + TF == PPCII::MO_GOT_TLSLD_PCREL_FLAG || + TF == PPCII::MO_GOT_TPREL_PCREL_FLAG || + TF == PPCII::MO_GOT_PCREL_FLAG; + } + + static bool hasTLSFlag(unsigned TF) { + return TF == PPCII::MO_TLSGD_FLAG || TF == PPCII::MO_TPREL_FLAG || + TF == PPCII::MO_TLSLD_FLAG || TF == PPCII::MO_TLSGDM_FLAG || + TF == PPCII::MO_GOT_TLSGD_PCREL_FLAG || + TF == PPCII::MO_GOT_TLSLD_PCREL_FLAG || + TF == PPCII::MO_GOT_TPREL_PCREL_FLAG || TF == PPCII::MO_TPREL_LO || + TF == PPCII::MO_TPREL_HA || TF == PPCII::MO_DTPREL_LO || + TF == PPCII::MO_TLSLD_LO || TF == PPCII::MO_TLS || + TF == PPCII::MO_TPREL_PCREL_FLAG || TF == PPCII::MO_TLS_PCREL_FLAG; + } + ScheduleHazardRecognizer * CreateTargetHazardRecognizer(const TargetSubtargetInfo *STI, const ScheduleDAG *DAG) const override; @@ -557,9 +584,6 @@ public: ArrayRef> getSerializableDirectMachineOperandTargetFlags() const override; - ArrayRef> - getSerializableBitmaskMachineOperandTargetFlags() const override; - // Expand VSX Memory Pseudo instruction to either a VSX or a FP instruction. bool expandVSXMemPseudo(MachineInstr &MI) const; diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td index a97062e0c643..6199785206b2 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td +++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td @@ -3437,6 +3437,23 @@ def crnot : OutPatFrag<(ops node:$in), def : Pat<(not i1:$in), (crnot $in)>; +// Pseudo-instructions for alternate assembly syntax (never used by codegen). +// These are aliases that require C++ handling to convert to the target +// instruction, while InstAliases can be handled directly by tblgen. +class PPCAsmPseudo + : Instruction { + let Namespace = "PPC"; + bit PPC64 = 0; // Default value, override with isPPC64 + + let OutOperandList = (outs); + let InOperandList = iops; + let Pattern = []; + let AsmString = asm; + let isAsmParserOnly = 1; + let isPseudo = 1; + let hasNoSchedulingInfo = 1; +} + // Prefixed instructions may require access to the above defs at a later // time so we include this after the def. include "PPCInstrP10.td" @@ -4458,23 +4475,6 @@ def ICBIEP : XForm_1a<31, 991, (outs), (ins (memrr $RA, $RB):$addr), "icbiep $a // PowerPC Assembler Instruction Aliases // -// Pseudo-instructions for alternate assembly syntax (never used by codegen). -// These are aliases that require C++ handling to convert to the target -// instruction, while InstAliases can be handled directly by tblgen. -class PPCAsmPseudo - : Instruction { - let Namespace = "PPC"; - bit PPC64 = 0; // Default value, override with isPPC64 - - let OutOperandList = (outs); - let InOperandList = iops; - let Pattern = []; - let AsmString = asm; - let isAsmParserOnly = 1; - let isPseudo = 1; - let hasNoSchedulingInfo = 1; -} - def : InstAlias<"sc", (SC 0)>; def : InstAlias<"sync", (SYNC 0)>, Requires<[HasSYNC]>; diff --git a/llvm/lib/Target/PowerPC/PPCInstrP10.td b/llvm/lib/Target/PowerPC/PPCInstrP10.td index f72a52d5b26f..d5a372e4dc10 100644 --- a/llvm/lib/Target/PowerPC/PPCInstrP10.td +++ b/llvm/lib/Target/PowerPC/PPCInstrP10.td @@ -575,33 +575,54 @@ class XForm_XT5_BI5 opcode, bits<10> xo, dag OOL, dag IOL, } multiclass MLS_DForm_R_SI34_RTA5_MEM_p opcode, dag OOL, dag IOL, - dag PCRel_IOL, string asmstr, + dag PCRel_IOL, dag PCRelOnly_IOL, + string asmstr, string asmstr_pcext, InstrItinClass itin> { def NAME : MLS_DForm_R_SI34_RTA5_MEM; def pc : MLS_DForm_R_SI34_RTA5_MEM, isPCRel; + let isAsmParserOnly = 1, hasNoSchedulingInfo = 1 in { + def nopc : MLS_DForm_R_SI34_RTA5_MEM; + let RA = 0 in + def onlypc : MLS_DForm_R_SI34_RTA5_MEM, isPCRel; + } } multiclass 8LS_DForm_R_SI34_RTA5_MEM_p opcode, dag OOL, dag IOL, - dag PCRel_IOL, string asmstr, + dag PCRel_IOL, dag PCRelOnly_IOL, + string asmstr, string asmstr_pcext, InstrItinClass itin> { def NAME : 8LS_DForm_R_SI34_RTA5_MEM; def pc : 8LS_DForm_R_SI34_RTA5_MEM, isPCRel; + let isAsmParserOnly = 1, hasNoSchedulingInfo = 1 in { + def nopc : 8LS_DForm_R_SI34_RTA5_MEM; + let RA = 0 in + def onlypc : 8LS_DForm_R_SI34_RTA5_MEM, isPCRel; + } } multiclass 8LS_DForm_R_SI34_XT6_RA5_MEM_p opcode, dag OOL, dag IOL, - dag PCRel_IOL, string asmstr, + dag PCRel_IOL, dag PCRelOnly_IOL, + string asmstr, string asmstr_pcext, InstrItinClass itin> { def NAME : 8LS_DForm_R_SI34_XT6_RA5_MEM; def pc : 8LS_DForm_R_SI34_XT6_RA5_MEM, isPCRel; + let isAsmParserOnly = 1, hasNoSchedulingInfo = 1 in { + def nopc : 8LS_DForm_R_SI34_XT6_RA5_MEM; + let RA = 0 in + def onlypc : 8LS_DForm_R_SI34_XT6_RA5_MEM, isPCRel; + } } def PrefixInstrs : Predicate<"Subtarget->hasPrefixInstrs()">; @@ -638,68 +659,88 @@ let Predicates = [PrefixInstrs] in { defm PLXV : 8LS_DForm_R_SI34_XT6_RA5_MEM_p<25, (outs vsrc:$XST), (ins (memri34 $D, $RA):$addr), (ins (memri34_pcrel $D, $RA):$addr), - "plxv $XST, $addr", IIC_LdStLFD>; + (ins s34imm_pcrel:$D), + "plxv $XST, $addr", "plxv $XST, $D", IIC_LdStLFD>; defm PLFS : MLS_DForm_R_SI34_RTA5_MEM_p<48, (outs f4rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plfs $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plfs $RST, $addr", + "plfs $RST, $D", IIC_LdStLFD>; defm PLFD : MLS_DForm_R_SI34_RTA5_MEM_p<50, (outs f8rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plfd $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plfd $RST, $addr", + "plfd $RST, $D", IIC_LdStLFD>; defm PLXSSP : 8LS_DForm_R_SI34_RTA5_MEM_p<43, (outs vfrc:$RST), (ins (memri34 $D, $RA):$addr), (ins (memri34_pcrel $D, $RA):$addr), - "plxssp $RST, $addr", IIC_LdStLFD>; + (ins s34imm_pcrel:$D), + "plxssp $RST, $addr", "plxssp $RST, $D", + IIC_LdStLFD>; defm PLXSD : 8LS_DForm_R_SI34_RTA5_MEM_p<42, (outs vfrc:$RST), (ins (memri34 $D, $RA):$addr), (ins (memri34_pcrel $D, $RA):$addr), - "plxsd $RST, $addr", IIC_LdStLFD>; + (ins s34imm_pcrel:$D), + "plxsd $RST, $addr", "plxsd $RST, $D", + IIC_LdStLFD>; let Interpretation64Bit = 1, isCodeGenOnly = 1 in { defm PLBZ8 : MLS_DForm_R_SI34_RTA5_MEM_p<34, (outs g8rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plbz $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plbz $RST, $addr", + "plbz $RST, $D", IIC_LdStLFD>; defm PLHZ8 : MLS_DForm_R_SI34_RTA5_MEM_p<40, (outs g8rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plhz $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plhz $RST, $addr", + "plhz $RST, $D", IIC_LdStLFD>; defm PLHA8 : MLS_DForm_R_SI34_RTA5_MEM_p<42, (outs g8rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plha $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plha $RST, $addr", + "plha $RST, $D", IIC_LdStLFD>; defm PLWA8 : 8LS_DForm_R_SI34_RTA5_MEM_p<41, (outs g8rc:$RST), (ins (memri34 $D, $RA):$addr), (ins (memri34_pcrel $D, $RA):$addr), - "plwa $RST, $addr", IIC_LdStLFD>; + (ins s34imm_pcrel:$D), + "plwa $RST, $addr", "plwa $RST, $D", IIC_LdStLFD>; defm PLWZ8 : MLS_DForm_R_SI34_RTA5_MEM_p<32, (outs g8rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plwz $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plwz $RST, $addr", + "plwz $RST, $D", IIC_LdStLFD>; } defm PLBZ : MLS_DForm_R_SI34_RTA5_MEM_p<34, (outs gprc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plbz $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plbz $RST, $addr", + "plbz $RST, $D", IIC_LdStLFD>; defm PLHZ : MLS_DForm_R_SI34_RTA5_MEM_p<40, (outs gprc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plhz $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plhz $RST, $addr", + "plhz $RST, $D", IIC_LdStLFD>; defm PLHA : MLS_DForm_R_SI34_RTA5_MEM_p<42, (outs gprc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plha $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plha $RST, $addr", + "plha $RST, $D", IIC_LdStLFD>; defm PLWZ : MLS_DForm_R_SI34_RTA5_MEM_p<32, (outs gprc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plwz $RST, $addr", - IIC_LdStLFD>; + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), "plwz $RST, $addr", + "plwz $RST, $D", IIC_LdStLFD>; defm PLWA : 8LS_DForm_R_SI34_RTA5_MEM_p<41, (outs gprc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plwa $RST, $addr", + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), + "plwa $RST, $addr", "plwa $RST, $D", IIC_LdStLFD>; defm PLD : 8LS_DForm_R_SI34_RTA5_MEM_p<57, (outs g8rc:$RST), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "pld $RST, $addr", + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), + "pld $RST, $addr", "pld $RST, $D", IIC_LdStLFD>; } @@ -707,53 +748,65 @@ let Predicates = [PrefixInstrs] in { defm PSTXV : 8LS_DForm_R_SI34_XT6_RA5_MEM_p<27, (outs), (ins vsrc:$XST, (memri34 $D, $RA):$addr), (ins vsrc:$XST, (memri34_pcrel $D, $RA):$addr), - "pstxv $XST, $addr", IIC_LdStLFD>; + (ins vsrc:$XST, s34imm_pcrel:$D), + "pstxv $XST, $addr", "pstxv $XST, $D", IIC_LdStLFD>; defm PSTFS : MLS_DForm_R_SI34_RTA5_MEM_p<52, (outs), (ins f4rc:$RST, (memri34 $D, $RA):$addr), (ins f4rc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstfs $RST, $addr", IIC_LdStLFD>; + (ins f4rc:$RST, s34imm_pcrel:$D), + "pstfs $RST, $addr", "pstfs $RST, $D", IIC_LdStLFD>; defm PSTFD : MLS_DForm_R_SI34_RTA5_MEM_p<54, (outs), (ins f8rc:$RST, (memri34 $D, $RA):$addr), (ins f8rc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstfd $RST, $addr", IIC_LdStLFD>; + (ins f8rc:$RST, s34imm_pcrel:$D), + "pstfd $RST, $addr", "pstfd $RST, $D", IIC_LdStLFD>; defm PSTXSSP : 8LS_DForm_R_SI34_RTA5_MEM_p<47, (outs), (ins vfrc:$RST, (memri34 $D, $RA):$addr), (ins vfrc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstxssp $RST, $addr", IIC_LdStLFD>; + (ins vfrc:$RST, s34imm_pcrel:$D), + "pstxssp $RST, $addr", "pstxssp $RST, $D", IIC_LdStLFD>; defm PSTXSD : 8LS_DForm_R_SI34_RTA5_MEM_p<46, (outs), (ins vfrc:$RST, (memri34 $D, $RA):$addr), (ins vfrc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstxsd $RST, $addr", IIC_LdStLFD>; + (ins vfrc:$RST, s34imm_pcrel:$D), + "pstxsd $RST, $addr", "pstxsd $RST, $D", IIC_LdStLFD>; let Interpretation64Bit = 1, isCodeGenOnly = 1 in { defm PSTB8 : MLS_DForm_R_SI34_RTA5_MEM_p<38, (outs), (ins g8rc:$RST, (memri34 $D, $RA):$addr), (ins g8rc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstb $RST, $addr", IIC_LdStLFD>; + (ins g8rc:$RST, s34imm_pcrel:$D), + "pstb $RST, $addr", "pstb $RST, $D", IIC_LdStLFD>; defm PSTH8 : MLS_DForm_R_SI34_RTA5_MEM_p<44, (outs), (ins g8rc:$RST, (memri34 $D, $RA):$addr), (ins g8rc:$RST, (memri34_pcrel $D, $RA):$addr), - "psth $RST, $addr", IIC_LdStLFD>; + (ins g8rc:$RST, s34imm_pcrel:$D), + "psth $RST, $addr", "psth $RST, $D", IIC_LdStLFD>; defm PSTW8 : MLS_DForm_R_SI34_RTA5_MEM_p<36, (outs), (ins g8rc:$RST, (memri34 $D, $RA):$addr), (ins g8rc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstw $RST, $addr", IIC_LdStLFD>; + (ins g8rc:$RST, s34imm_pcrel:$D), + "pstw $RST, $addr", "pstw $RST, $D", IIC_LdStLFD>; } defm PSTB : MLS_DForm_R_SI34_RTA5_MEM_p<38, (outs), (ins gprc:$RST, (memri34 $D, $RA):$addr), (ins gprc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstb $RST, $addr", IIC_LdStLFD>; + (ins gprc:$RST, s34imm_pcrel:$D), + "pstb $RST, $addr", "pstb $RST, $D", IIC_LdStLFD>; defm PSTH : MLS_DForm_R_SI34_RTA5_MEM_p<44, (outs), (ins gprc:$RST, (memri34 $D, $RA):$addr), (ins gprc:$RST, (memri34_pcrel $D, $RA):$addr), - "psth $RST, $addr", IIC_LdStLFD>; + (ins gprc:$RST, s34imm_pcrel:$D), + "psth $RST, $addr", "psth $RST, $D", IIC_LdStLFD>; defm PSTW : MLS_DForm_R_SI34_RTA5_MEM_p<36, (outs), (ins gprc:$RST, (memri34 $D, $RA):$addr), (ins gprc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstw $RST, $addr", IIC_LdStLFD>; + (ins gprc:$RST, s34imm_pcrel:$D), + "pstw $RST, $addr", "pstw $RST, $D", IIC_LdStLFD>; defm PSTD : 8LS_DForm_R_SI34_RTA5_MEM_p<61, (outs), (ins g8rc:$RST, (memri34 $D, $RA):$addr), (ins g8rc:$RST, (memri34_pcrel $D, $RA):$addr), - "pstd $RST, $addr", IIC_LdStLFD>; + (ins g8rc:$RST, s34imm_pcrel:$D), + "pstd $RST, $addr", "pstd $RST, $D", IIC_LdStLFD>; } } @@ -812,13 +865,20 @@ class 8LS_DForm_R_XTp5_SI34_MEM opcode, dag OOL, dag IOL, string asmstr, } multiclass 8LS_DForm_R_XTp5_SI34_MEM_p opcode, dag OOL, - dag IOL, dag PCRel_IOL, - string asmstr, InstrItinClass itin> { + dag IOL, dag PCRel_IOL, dag PCRelOnly_IOL, + string asmstr, string asmstr_pcext, + InstrItinClass itin> { def NAME : 8LS_DForm_R_XTp5_SI34_MEM; def pc : 8LS_DForm_R_XTp5_SI34_MEM, isPCRel; + let isAsmParserOnly = 1, hasNoSchedulingInfo = 1 in { + def nopc : 8LS_DForm_R_XTp5_SI34_MEM; + let RA = 0 in + def onlypc : 8LS_DForm_R_XTp5_SI34_MEM, isPCRel; + } } @@ -1079,7 +1139,9 @@ let mayLoad = 0, mayStore = 1, Predicates = [PairedVectorMemops] in { let mayLoad = 1, mayStore = 0, Predicates = [PairedVectorMemops, PrefixInstrs] in { defm PLXVP : 8LS_DForm_R_XTp5_SI34_MEM_p<58, (outs vsrprc:$XTp), (ins (memri34 $D, $RA):$addr), - (ins (memri34_pcrel $D, $RA):$addr), "plxvp $XTp, $addr", + (ins (memri34_pcrel $D, $RA):$addr), + (ins s34imm_pcrel:$D), + "plxvp $XTp, $addr", "plxvp $XTp, $D", IIC_LdStLFD>; } @@ -1087,7 +1149,8 @@ let mayLoad = 0, mayStore = 1, Predicates = [PairedVectorMemops, PrefixInstrs] i defm PSTXVP : 8LS_DForm_R_XTp5_SI34_MEM_p<62, (outs), (ins vsrprc:$XTp, (memri34 $D, $RA):$addr), (ins vsrprc:$XTp, (memri34_pcrel $D, $RA):$addr), - "pstxvp $XTp, $addr", IIC_LdStLFD>; + (ins vsrprc:$XTp, s34imm_pcrel:$D), + "pstxvp $XTp, $addr", "pstxvp $XTp, $D", IIC_LdStLFD>; } let Predicates = [PairedVectorMemops] in { @@ -1855,6 +1918,13 @@ let Predicates = [IsISA3_1, HasVSX] in { [(set f128:$RST, (PPCxsminc f128:$RA, f128:$RB))]>; } +let Predicates = [IsISA3_1] in { + def WAITP10 : XForm_IMM2_IMM2<31, 30, (outs), (ins u2imm:$L, u2imm:$PL), + "wait $L $PL", IIC_LdStLoad, []>; + def SYNCP10 : XForm_IMM3_IMM2<31, 598, (outs), (ins u3imm:$L, u2imm:$SC), + "sync $L, $SC", IIC_LdStSync, []>; +} + // Multiclass defining patterns for Set Boolean Extension Reverse Instructions. // This is analogous to the CRNotPat multiclass but specifically for Power10 // and newer subtargets since the extended forms use Set Boolean instructions. @@ -2386,3 +2456,50 @@ let AddedComplexity = 400, Predicates = [IsISA3_1, HasVSX, IsBigEndian] in { def : Pat<(v2i64 (insertelt v2i64:$vDi, i64:$rA, Idx)), (VINSD $vDi, !mul(Idx, 8), $rA)>; } + + +//===----------------------------------------------------------------------===// +// PowerPC ISA 3.1 Extended Mnemonics. +// + +let Predicates = [IsISA3_1] in { + def : InstAlias<"wait", (WAITP10 0, 0)>; + def : InstAlias<"wait 0", (WAITP10 0, 0), 0>; + def : InstAlias<"wait 1", (WAITP10 1, 0), 0>; + def : InstAlias<"waitrsv", (WAITP10 1, 0)>; + def : InstAlias<"pause_short", (WAITP10 2, 0), 0>; + + def : InstAlias<"sync", (SYNCP10 0, 0)>; + def : InstAlias<"hwsync", (SYNCP10 0, 0), 0>; + def : InstAlias<"wsync", (SYNCP10 1, 0), 0>; + def : InstAlias<"ptesync", (SYNCP10 2, 0)>; + def : InstAlias<"phwsync", (SYNCP10 4, 0)>; + def : InstAlias<"plwsync", (SYNCP10 5, 0)>; + def : InstAlias<"sync $L", (SYNCP10 u3imm:$L, 0)>; + def : InstAlias<"stncisync", (SYNCP10 1, 1)>; + def : InstAlias<"stcisync", (SYNCP10 0, 2)>; + def : InstAlias<"stsync", (SYNCP10 0, 3)>; + + def : InstAlias<"paddi $RT, $RA, $SI", (PADDI8 g8rc:$RT, g8rc_nox0:$RA, s34imm:$SI)>; +} + +let Predicates = [IsISA3_1, PrefixInstrs], isAsmParserOnly = 1, hasNoSchedulingInfo = 1 in { + let Interpretation64Bit = 1 in { + def PLA8 : MLS_DForm_SI34_RT5<14, (outs g8rc:$RT), + (ins g8rc_nox0:$RA, s34imm:$SI), + "pla $RT, ${SI} ${RA}", IIC_IntSimple, []>; + def PLA8pc : MLS_DForm_SI34_RT5<14, (outs g8rc:$RT), + (ins s34imm_pcrel:$SI), + "pla $RT, $SI", IIC_IntSimple, []>, isPCRel; + } + + def PSUBI : PPCAsmPseudo<"psubi $RT, $RA, $SI", + (ins g8rc:$RT, g8rc_nox0:$RA, s34imm:$SI)>; + + def PLA : MLS_DForm_SI34_RT5<14, (outs gprc:$RT), + (ins gprc_nor0:$RA, s34imm:$SI), + "pla $RT, ${SI} ${RA}", IIC_IntSimple, []>; + def PLApc : MLS_DForm_SI34_RT5<14, (outs gprc:$RT), + (ins s34imm_pcrel:$SI), + "pla $RT, $SI", IIC_IntSimple, []>, isPCRel; +} diff --git a/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp b/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp index c9797fdc2286..9a3ca5a78293 100644 --- a/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp +++ b/llvm/lib/Target/PowerPC/PPCMCInstLower.cpp @@ -54,7 +54,7 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol, MCContext &Ctx = Printer.OutContext; MCSymbolRefExpr::VariantKind RefKind = MCSymbolRefExpr::VK_None; - unsigned access = MO.getTargetFlags() & PPCII::MO_ACCESS_MASK; + unsigned access = MO.getTargetFlags(); switch (access) { case PPCII::MO_TPREL_LO: @@ -73,9 +73,10 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol, RefKind = MCSymbolRefExpr::VK_PPC_TOC_LO; break; case PPCII::MO_TLS: - bool IsPCRel = (MO.getTargetFlags() & ~access) == PPCII::MO_PCREL_FLAG; - RefKind = IsPCRel ? MCSymbolRefExpr::VK_PPC_TLS_PCREL - : MCSymbolRefExpr::VK_PPC_TLS; + RefKind = MCSymbolRefExpr::VK_PPC_TLS; + break; + case PPCII::MO_TLS_PCREL_FLAG: + RefKind = MCSymbolRefExpr::VK_PPC_TLS_PCREL; break; } @@ -85,9 +86,9 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol, RefKind = MCSymbolRefExpr::VK_PLT; else if (MO.getTargetFlags() == PPCII::MO_PCREL_FLAG) RefKind = MCSymbolRefExpr::VK_PCREL; - else if (MO.getTargetFlags() == (PPCII::MO_PCREL_FLAG | PPCII::MO_GOT_FLAG)) + else if (MO.getTargetFlags() == PPCII::MO_GOT_PCREL_FLAG) RefKind = MCSymbolRefExpr::VK_PPC_GOT_PCREL; - else if (MO.getTargetFlags() == (PPCII::MO_PCREL_FLAG | PPCII::MO_TPREL_FLAG)) + else if (MO.getTargetFlags() == PPCII::MO_TPREL_PCREL_FLAG) RefKind = MCSymbolRefExpr::VK_TPREL; else if (MO.getTargetFlags() == PPCII::MO_GOT_TLSGD_PCREL_FLAG) RefKind = MCSymbolRefExpr::VK_PPC_GOT_TLSGD_PCREL; @@ -138,7 +139,9 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol, Ctx); // Subtract off the PIC base if required. - if (MO.getTargetFlags() & PPCII::MO_PIC_FLAG) { + if (MO.getTargetFlags() == PPCII::MO_PIC_FLAG || + MO.getTargetFlags() == PPCII::MO_PIC_HA_FLAG || + MO.getTargetFlags() == PPCII::MO_PIC_LO_FLAG) { const MachineFunction *MF = MO.getParent()->getParent()->getParent(); const MCExpr *PB = MCSymbolRefExpr::create(MF->getPICBaseSymbol(), Ctx); @@ -148,9 +151,11 @@ static MCOperand GetSymbolRef(const MachineOperand &MO, const MCSymbol *Symbol, // Add ha16() / lo16() markers if required. switch (access) { case PPCII::MO_LO: + case PPCII::MO_PIC_LO_FLAG: Expr = PPCMCExpr::createLo(Expr, Ctx); break; case PPCII::MO_HA: + case PPCII::MO_PIC_HA_FLAG: Expr = PPCMCExpr::createHa(Expr, Ctx); break; } diff --git a/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp b/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp index 6ad69fffea87..6e3bf26a598a 100644 --- a/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp +++ b/llvm/lib/Target/PowerPC/PPCPreEmitPeephole.cpp @@ -239,7 +239,7 @@ static bool hasPCRelativeForm(MachineInstr &Use) { return false; // Finally return true only if the GOT flag is present. - return (SymbolOp.getTargetFlags() & PPCII::MO_GOT_FLAG); + return PPCInstrInfo::hasGOTFlag(SymbolOp.getTargetFlags()); } bool addLinkerOpt(MachineBasicBlock &MBB, const TargetRegisterInfo *TRI) { diff --git a/llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp b/llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp index 8120975c4fb2..81f078ab246e 100644 --- a/llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp +++ b/llvm/lib/Target/PowerPC/PPCTOCRegDeps.cpp @@ -100,7 +100,7 @@ protected: return true; for (const MachineOperand &MO : MI.operands()) { - if ((MO.getTargetFlags() & PPCII::MO_ACCESS_MASK) == PPCII::MO_TOC_LO) + if (MO.getTargetFlags() == PPCII::MO_TOC_LO) return true; } diff --git a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp index aa385d7c3b20..062b53e24a0d 100644 --- a/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCTargetTransformInfo.cpp @@ -18,7 +18,6 @@ #include "llvm/IR/ProfDataUtils.h" #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" -#include "llvm/Support/KnownBits.h" #include "llvm/Transforms/InstCombine/InstCombiner.h" #include "llvm/Transforms/Utils/Local.h" #include diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp index 570b9802a2f5..8f03a7ac41d3 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp +++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp @@ -13,6 +13,7 @@ #include "RISCVLegalizerInfo.h" #include "RISCVMachineFunctionInfo.h" #include "RISCVSubtarget.h" +#include "llvm/CodeGen/GlobalISel/GenericMachineInstrs.h" #include "llvm/CodeGen/GlobalISel/LegalizerHelper.h" #include "llvm/CodeGen/GlobalISel/MachineIRBuilder.h" #include "llvm/CodeGen/MachineRegisterInfo.h" @@ -317,9 +318,60 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST) getActionDefinitionsBuilder(G_VASTART).customFor({p0}); + // va_list must be a pointer, but most sized types are pretty easy to handle + // as the destination. + getActionDefinitionsBuilder(G_VAARG) + // TODO: Implement narrowScalar and widenScalar for G_VAARG for types + // outside the [s32, sXLen] range. + .clampScalar(0, s32, sXLen) + .lowerForCartesianProduct({s32, sXLen, p0}, {p0}); + getLegacyLegalizerInfo().computeTables(); } +static Type *getTypeForLLT(LLT Ty, LLVMContext &C) { + if (Ty.isVector()) + return FixedVectorType::get(IntegerType::get(C, Ty.getScalarSizeInBits()), + Ty.getNumElements()); + return IntegerType::get(C, Ty.getSizeInBits()); +} + +bool RISCVLegalizerInfo::legalizeIntrinsic(LegalizerHelper &Helper, + MachineInstr &MI) const { + Intrinsic::ID IntrinsicID = cast(MI).getIntrinsicID(); + switch (IntrinsicID) { + default: + return false; + case Intrinsic::vacopy: { + // vacopy arguments must be legal because of the intrinsic signature. + // No need to check here. + + MachineIRBuilder &MIRBuilder = Helper.MIRBuilder; + MachineRegisterInfo &MRI = *MIRBuilder.getMRI(); + MachineFunction &MF = *MI.getMF(); + const DataLayout &DL = MIRBuilder.getDataLayout(); + LLVMContext &Ctx = MF.getFunction().getContext(); + + Register DstLst = MI.getOperand(1).getReg(); + LLT PtrTy = MRI.getType(DstLst); + + // Load the source va_list + Align Alignment = DL.getABITypeAlign(getTypeForLLT(PtrTy, Ctx)); + MachineMemOperand *LoadMMO = MF.getMachineMemOperand( + MachinePointerInfo(), MachineMemOperand::MOLoad, PtrTy, Alignment); + auto Tmp = MIRBuilder.buildLoad(PtrTy, MI.getOperand(2), *LoadMMO); + + // Store the result in the destination va_list + MachineMemOperand *StoreMMO = MF.getMachineMemOperand( + MachinePointerInfo(), MachineMemOperand::MOStore, PtrTy, Alignment); + MIRBuilder.buildStore(DstLst, Tmp, *StoreMMO); + + MI.eraseFromParent(); + return true; + } + } +} + bool RISCVLegalizerInfo::legalizeShlAshrLshr( MachineInstr &MI, MachineIRBuilder &MIRBuilder, GISelChangeObserver &Observer) const { @@ -354,7 +406,7 @@ bool RISCVLegalizerInfo::legalizeVAStart(MachineInstr &MI, LLT AddrTy = MIRBuilder.getMRI()->getType(MI.getOperand(0).getReg()); auto FINAddr = MIRBuilder.buildFrameIndex(AddrTy, FI); assert(MI.hasOneMemOperand()); - MIRBuilder.buildStore(MI.getOperand(0).getReg(), FINAddr, + MIRBuilder.buildStore(FINAddr, MI.getOperand(0).getReg(), *MI.memoperands()[0]); MI.eraseFromParent(); return true; diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h index 246ea90dcd74..48c36976501f 100644 --- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h +++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.h @@ -32,6 +32,9 @@ public: bool legalizeCustom(LegalizerHelper &Helper, MachineInstr &MI) const override; + bool legalizeIntrinsic(LegalizerHelper &Helper, + MachineInstr &MI) const override; + private: bool legalizeShlAshrLshr(MachineInstr &MI, MachineIRBuilder &MIRBuilder, GISelChangeObserver &Observer) const; diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp index dfc3c9e9908d..716fb67c5824 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.cpp @@ -108,7 +108,8 @@ RISCVAsmBackend::getFixupKindInfo(MCFixupKind Kind) const { // necessary for correctness as offsets may change during relaxation. bool RISCVAsmBackend::shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) { + const MCValue &Target, + const MCSubtargetInfo *STI) { if (Fixup.getKind() >= FirstLiteralRelocationKind) return true; switch (Fixup.getTargetKind()) { @@ -128,7 +129,7 @@ bool RISCVAsmBackend::shouldForceRelocation(const MCAssembler &Asm, return true; } - return STI.hasFeature(RISCV::FeatureRelax) || ForceRelocs; + return STI->hasFeature(RISCV::FeatureRelax) || ForceRelocs; } bool RISCVAsmBackend::fixupNeedsRelaxationAdvanced(const MCFixup &Fixup, @@ -514,8 +515,8 @@ static uint64_t adjustFixupValue(const MCFixup &Fixup, uint64_t Value, bool RISCVAsmBackend::evaluateTargetFixup( const MCAssembler &Asm, const MCAsmLayout &Layout, const MCFixup &Fixup, - const MCFragment *DF, const MCValue &Target, uint64_t &Value, - bool &WasForced) { + const MCFragment *DF, const MCValue &Target, const MCSubtargetInfo *STI, + uint64_t &Value, bool &WasForced) { const MCFixup *AUIPCFixup; const MCFragment *AUIPCDF; MCValue AUIPCTarget; @@ -565,7 +566,7 @@ bool RISCVAsmBackend::evaluateTargetFixup( Value = Layout.getSymbolOffset(SA) + AUIPCTarget.getConstant(); Value -= Layout.getFragmentOffset(AUIPCDF) + AUIPCFixup->getOffset(); - if (shouldForceRelocation(Asm, *AUIPCFixup, AUIPCTarget)) { + if (shouldForceRelocation(Asm, *AUIPCFixup, AUIPCTarget, STI)) { WasForced = true; return false; } diff --git a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h index 99b0d7b223b9..2ad6534ac8bc 100644 --- a/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h +++ b/llvm/lib/Target/RISCV/MCTargetDesc/RISCVAsmBackend.h @@ -50,8 +50,8 @@ public: bool evaluateTargetFixup(const MCAssembler &Asm, const MCAsmLayout &Layout, const MCFixup &Fixup, const MCFragment *DF, - const MCValue &Target, uint64_t &Value, - bool &WasForced) override; + const MCValue &Target, const MCSubtargetInfo *STI, + uint64_t &Value, bool &WasForced) override; bool handleAddSubRelocations(const MCAsmLayout &Layout, const MCFragment &F, const MCFixup &Fixup, const MCValue &Target, @@ -66,7 +66,8 @@ public: createObjectTargetWriter() const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; bool fixupNeedsRelaxation(const MCFixup &Fixup, uint64_t Value, const MCRelaxableFragment *DF, diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h index ad1713cad64c..9eb18099894b 100644 --- a/llvm/lib/Target/RISCV/RISCV.h +++ b/llvm/lib/Target/RISCV/RISCV.h @@ -18,13 +18,8 @@ #include "llvm/Target/TargetMachine.h" namespace llvm { -class AsmPrinter; class FunctionPass; class InstructionSelector; -class MCInst; -class MCOperand; -class MachineInstr; -class MachineOperand; class PassRegistry; class RISCVRegisterBankInfo; class RISCVSubtarget; diff --git a/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp b/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp index 7099f36dcd43..df607236f7d5 100644 --- a/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp +++ b/llvm/lib/Target/RISCV/RISCVDeadRegisterDefinitions.cpp @@ -28,10 +28,7 @@ class RISCVDeadRegisterDefinitions : public MachineFunctionPass { public: static char ID; - RISCVDeadRegisterDefinitions() : MachineFunctionPass(ID) { - initializeRISCVDeadRegisterDefinitionsPass( - *PassRegistry::getPassRegistry()); - } + RISCVDeadRegisterDefinitions() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; void getAnalysisUsage(AnalysisUsage &AU) const override { AU.setPreservesCFG(); diff --git a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp index aa1bdbd8ba9c..24a13f93af88 100644 --- a/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp +++ b/llvm/lib/Target/RISCV/RISCVExpandPseudoInsts.cpp @@ -34,9 +34,7 @@ public: const RISCVInstrInfo *TII; static char ID; - RISCVExpandPseudo() : MachineFunctionPass(ID) { - initializeRISCVExpandPseudoPass(*PassRegistry::getPassRegistry()); - } + RISCVExpandPseudo() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; @@ -381,9 +379,7 @@ public: const RISCVInstrInfo *TII; static char ID; - RISCVPreRAExpandPseudo() : MachineFunctionPass(ID) { - initializeRISCVPreRAExpandPseudoPass(*PassRegistry::getPassRegistry()); - } + RISCVPreRAExpandPseudo() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; diff --git a/llvm/lib/Target/RISCV/RISCVFeatures.td b/llvm/lib/Target/RISCV/RISCVFeatures.td index 7d142d38d0f9..294927aecb94 100644 --- a/llvm/lib/Target/RISCV/RISCVFeatures.td +++ b/llvm/lib/Target/RISCV/RISCVFeatures.td @@ -970,6 +970,16 @@ def TuneLUIADDIFusion : SubtargetFeature<"lui-addi-fusion", "HasLUIADDIFusion", "true", "Enable LUI+ADDI macrofusion">; +def TuneAUIPCADDIFusion + : SubtargetFeature<"auipc-addi-fusion", "HasAUIPCADDIFusion", + "true", "Enable AUIPC+ADDI macrofusion">; +def TuneShiftedZExtFusion + : SubtargetFeature<"shifted-zext-fusion", "HasShiftedZExtFusion", + "true", "Enable SLLI+SRLI to be fused when computing (shifted) zero extension">; +def TuneLDADDFusion + : SubtargetFeature<"ld-add-fusion", "HasLDADDFusion", + "true", "Enable LD+ADD macrofusion.">; + def TuneNoDefaultUnroll : SubtargetFeature<"no-default-unroll", "EnableDefaultUnroll", "false", "Disable default unroll preference.">; @@ -987,9 +997,12 @@ def TuneSiFive7 : SubtargetFeature<"sifive7", "RISCVProcFamily", "SiFive7", [TuneNoDefaultUnroll, TuneShortForwardBranchOpt]>; -def TuneVentanaVeyron : SubtargetFeature<"ventana-veyron", "RISCVProcFamily", "VentanaVeyron", - "Ventana-Veyron Series processors", - [TuneLUIADDIFusion]>; +def TuneVeyronFusions : SubtargetFeature<"ventana-veyron", "RISCVProcFamily", "VentanaVeyron", + "Ventana Veyron-Series processors", + [TuneLUIADDIFusion, + TuneAUIPCADDIFusion, + TuneShiftedZExtFusion, + TuneLDADDFusion]>; // Assume that lock-free native-width atomics are available, even if the target // and operating system combination would not usually provide them. The user diff --git a/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp b/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp index 871fc76d7d98..6ee006525df5 100644 --- a/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp +++ b/llvm/lib/Target/RISCV/RISCVFoldMasks.cpp @@ -36,9 +36,7 @@ public: const TargetInstrInfo *TII; MachineRegisterInfo *MRI; const TargetRegisterInfo *TRI; - RISCVFoldMasks() : MachineFunctionPass(ID) { - initializeRISCVFoldMasksPass(*PassRegistry::getPassRegistry()); - } + RISCVFoldMasks() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; MachineFunctionProperties getRequiredProperties() const override { diff --git a/llvm/lib/Target/RISCV/RISCVInsertReadWriteCSR.cpp b/llvm/lib/Target/RISCV/RISCVInsertReadWriteCSR.cpp index acd19bf1b8a1..b807abcc5681 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertReadWriteCSR.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertReadWriteCSR.cpp @@ -31,9 +31,7 @@ class RISCVInsertReadWriteCSR : public MachineFunctionPass { public: static char ID; - RISCVInsertReadWriteCSR() : MachineFunctionPass(ID) { - initializeRISCVInsertReadWriteCSRPass(*PassRegistry::getPassRegistry()); - } + RISCVInsertReadWriteCSR() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; diff --git a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp index 323a92cfb8c8..ed3489f6db1c 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertVSETVLI.cpp @@ -742,9 +742,7 @@ class RISCVInsertVSETVLI : public MachineFunctionPass { public: static char ID; - RISCVInsertVSETVLI() : MachineFunctionPass(ID) { - initializeRISCVInsertVSETVLIPass(*PassRegistry::getPassRegistry()); - } + RISCVInsertVSETVLI() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; void getAnalysisUsage(AnalysisUsage &AU) const override { diff --git a/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp b/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp index 70c2415465f8..de2227f82192 100644 --- a/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp +++ b/llvm/lib/Target/RISCV/RISCVInsertWriteVXRM.cpp @@ -171,9 +171,7 @@ class RISCVInsertWriteVXRM : public MachineFunctionPass { public: static char ID; - RISCVInsertWriteVXRM() : MachineFunctionPass(ID) { - initializeRISCVInsertWriteVXRMPass(*PassRegistry::getPassRegistry()); - } + RISCVInsertWriteVXRM() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td index 127d3080491d..5e06422cf9ad 100644 --- a/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td +++ b/llvm/lib/Target/RISCV/RISCVInstrInfoVPseudos.td @@ -81,9 +81,9 @@ def riscv_vmv_x_s : SDNode<"RISCVISD::VMV_X_S", def riscv_read_vlenb : SDNode<"RISCVISD::READ_VLENB", SDTypeProfile<1, 0, [SDTCisVT<0, XLenVT>]>>; -// Operand that is allowed to be a register or a 5 bit immediate. -// This allows us to pick between VSETIVLI and VSETVLI opcodes using the same -// pseudo instructions. +// Operand that is allowed to be a register other than X0, a 5 bit unsigned +// immediate, or -1. -1 means VLMAX. This allows us to pick between VSETIVLI and +// VSETVLI opcodes using the same pseudo instructions. def AVL : RegisterOperand { let OperandNamespace = "RISCVOp"; let OperandType = "OPERAND_AVL"; diff --git a/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp b/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp index 02a8d5c18fe1..02ea5270823d 100644 --- a/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp +++ b/llvm/lib/Target/RISCV/RISCVMacroFusion.cpp @@ -18,6 +18,101 @@ using namespace llvm; +static bool checkRegisters(Register FirstDest, const MachineInstr &SecondMI) { + if (!SecondMI.getOperand(1).isReg()) + return false; + + if (SecondMI.getOperand(1).getReg() != FirstDest) + return false; + + // If the input is virtual make sure this is the only user. + if (FirstDest.isVirtual()) { + auto &MRI = SecondMI.getMF()->getRegInfo(); + return MRI.hasOneNonDBGUse(FirstDest); + } + + return SecondMI.getOperand(0).getReg() == FirstDest; +} + +// Fuse load with add: +// add rd, rs1, rs2 +// ld rd, 0(rd) +static bool isLDADD(const MachineInstr *FirstMI, const MachineInstr &SecondMI) { + if (SecondMI.getOpcode() != RISCV::LD) + return false; + + if (!SecondMI.getOperand(2).isImm()) + return false; + + if (SecondMI.getOperand(2).getImm() != 0) + return false; + + // Given SecondMI, when FirstMI is unspecified, we must return + // if SecondMI may be part of a fused pair at all. + if (!FirstMI) + return true; + + if (FirstMI->getOpcode() != RISCV::ADD) + return true; + + return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI); +} + +// Fuse these patterns: +// +// slli rd, rs1, 32 +// srli rd, rd, x +// where 0 <= x <= 32 +// +// and +// +// slli rd, rs1, 48 +// srli rd, rd, x +static bool isShiftedZExt(const MachineInstr *FirstMI, + const MachineInstr &SecondMI) { + if (SecondMI.getOpcode() != RISCV::SRLI) + return false; + + if (!SecondMI.getOperand(2).isImm()) + return false; + + unsigned SRLIImm = SecondMI.getOperand(2).getImm(); + bool IsShiftBy48 = SRLIImm == 48; + if (SRLIImm > 32 && !IsShiftBy48) + return false; + + // Given SecondMI, when FirstMI is unspecified, we must return + // if SecondMI may be part of a fused pair at all. + if (!FirstMI) + return true; + + if (FirstMI->getOpcode() != RISCV::SLLI) + return false; + + unsigned SLLIImm = FirstMI->getOperand(2).getImm(); + if (IsShiftBy48 ? (SLLIImm != 48) : (SLLIImm != 32)) + return false; + + return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI); +} + +// Fuse AUIPC followed by ADDI +// auipc rd, imm20 +// addi rd, rd, imm12 +static bool isAUIPCADDI(const MachineInstr *FirstMI, + const MachineInstr &SecondMI) { + if (SecondMI.getOpcode() != RISCV::ADDI) + return false; + // Assume the 1st instr to be a wildcard if it is unspecified. + if (!FirstMI) + return true; + + if (FirstMI->getOpcode() != RISCV::AUIPC) + return false; + + return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI); +} + // Fuse LUI followed by ADDI or ADDIW. // rd = imm[31:0] which decomposes to // lui rd, imm[31:12] @@ -27,7 +122,6 @@ static bool isLUIADDI(const MachineInstr *FirstMI, if (SecondMI.getOpcode() != RISCV::ADDI && SecondMI.getOpcode() != RISCV::ADDIW) return false; - // Assume the 1st instr to be a wildcard if it is unspecified. if (!FirstMI) return true; @@ -35,21 +129,7 @@ static bool isLUIADDI(const MachineInstr *FirstMI, if (FirstMI->getOpcode() != RISCV::LUI) return false; - Register FirstDest = FirstMI->getOperand(0).getReg(); - - // Destination of LUI should be the ADDI(W) source register. - if (SecondMI.getOperand(1).getReg() != FirstDest) - return false; - - // If the input is virtual make sure this is the only user. - if (FirstDest.isVirtual()) { - auto &MRI = SecondMI.getMF()->getRegInfo(); - return MRI.hasOneNonDBGUse(FirstDest); - } - - // If the FirstMI destination is non-virtual, it should match the SecondMI - // destination. - return SecondMI.getOperand(0).getReg() == FirstDest; + return checkRegisters(FirstMI->getOperand(0).getReg(), SecondMI); } static bool shouldScheduleAdjacent(const TargetInstrInfo &TII, @@ -61,6 +141,15 @@ static bool shouldScheduleAdjacent(const TargetInstrInfo &TII, if (ST.hasLUIADDIFusion() && isLUIADDI(FirstMI, SecondMI)) return true; + if (ST.hasAUIPCADDIFusion() && isAUIPCADDI(FirstMI, SecondMI)) + return true; + + if (ST.hasShiftedZExtFusion() && isShiftedZExt(FirstMI, SecondMI)) + return true; + + if (ST.hasLDADDFusion() && isLDADD(FirstMI, SecondMI)) + return true; + return false; } diff --git a/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp b/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp index 79da67d06e18..ff21fe1d4064 100644 --- a/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp +++ b/llvm/lib/Target/RISCV/RISCVMakeCompressible.cpp @@ -84,9 +84,7 @@ struct RISCVMakeCompressibleOpt : public MachineFunctionPass { bool runOnMachineFunction(MachineFunction &Fn) override; - RISCVMakeCompressibleOpt() : MachineFunctionPass(ID) { - initializeRISCVMakeCompressibleOptPass(*PassRegistry::getPassRegistry()); - } + RISCVMakeCompressibleOpt() : MachineFunctionPass(ID) {} StringRef getPassName() const override { return RISCV_COMPRESS_INSTRS_NAME; } }; diff --git a/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp b/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp index 097e12a2cd59..ae46d5554d35 100644 --- a/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp +++ b/llvm/lib/Target/RISCV/RISCVMergeBaseOffset.cpp @@ -19,7 +19,6 @@ #include "llvm/Support/Debug.h" #include "llvm/Target/TargetOptions.h" #include -#include using namespace llvm; #define DEBUG_TYPE "riscv-merge-base-offset" diff --git a/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp b/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp index 934a2a0f8e1a..3c5462057b28 100644 --- a/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp +++ b/llvm/lib/Target/RISCV/RISCVMoveMerger.cpp @@ -22,9 +22,7 @@ namespace { struct RISCVMoveMerge : public MachineFunctionPass { static char ID; - RISCVMoveMerge() : MachineFunctionPass(ID) { - initializeRISCVMoveMergePass(*PassRegistry::getPassRegistry()); - } + RISCVMoveMerge() : MachineFunctionPass(ID) {} const RISCVInstrInfo *TII; const TargetRegisterInfo *TRI; diff --git a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp index b667a2b7a11b..2c2b34bb5b77 100644 --- a/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp +++ b/llvm/lib/Target/RISCV/RISCVOptWInstrs.cpp @@ -53,9 +53,7 @@ class RISCVOptWInstrs : public MachineFunctionPass { public: static char ID; - RISCVOptWInstrs() : MachineFunctionPass(ID) { - initializeRISCVOptWInstrsPass(*PassRegistry::getPassRegistry()); - } + RISCVOptWInstrs() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; bool removeSExtWInstrs(MachineFunction &MF, const RISCVInstrInfo &TII, @@ -368,6 +366,18 @@ static bool isSignExtendingOpW(const MachineInstr &MI, return MI.getOperand(1).getReg() == RISCV::X0; case RISCV::PseudoAtomicLoadNand32: return true; + case RISCV::PseudoVMV_X_S_MF8: + case RISCV::PseudoVMV_X_S_MF4: + case RISCV::PseudoVMV_X_S_MF2: + case RISCV::PseudoVMV_X_S_M1: + case RISCV::PseudoVMV_X_S_M2: + case RISCV::PseudoVMV_X_S_M4: + case RISCV::PseudoVMV_X_S_M8: { + // vmv.x.s has at least 33 sign bits if log2(sew) <= 5. + int64_t Log2SEW = MI.getOperand(2).getImm(); + assert(Log2SEW >= 3 && Log2SEW <= 6 && "Unexpected Log2SEW"); + return Log2SEW <= 5; + } } return false; diff --git a/llvm/lib/Target/RISCV/RISCVProcessors.td b/llvm/lib/Target/RISCV/RISCVProcessors.td index 90ba99d3f845..58989fd716fa 100644 --- a/llvm/lib/Target/RISCV/RISCVProcessors.td +++ b/llvm/lib/Target/RISCV/RISCVProcessors.td @@ -254,7 +254,7 @@ def VENTANA_VEYRON_V1 : RISCVProcessorModel<"veyron-v1", FeatureStdExtZicbop, FeatureStdExtZicboz, FeatureVendorXVentanaCondOps], - [TuneVentanaVeyron]>; + [TuneVeyronFusions]>; def XIANGSHAN_NANHU : RISCVProcessorModel<"xiangshan-nanhu", NoSchedModel, diff --git a/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp b/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp index c2c795ec946a..009dcf57f46d 100644 --- a/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp +++ b/llvm/lib/Target/RISCV/RISCVPushPopOptimizer.cpp @@ -22,9 +22,7 @@ namespace { struct RISCVPushPopOpt : public MachineFunctionPass { static char ID; - RISCVPushPopOpt() : MachineFunctionPass(ID) { - initializeRISCVPushPopOptPass(*PassRegistry::getPassRegistry()); - } + RISCVPushPopOpt() : MachineFunctionPass(ID) {} const RISCVInstrInfo *TII; const TargetRegisterInfo *TRI; diff --git a/llvm/lib/Target/RISCV/RISCVRVVInitUndef.cpp b/llvm/lib/Target/RISCV/RISCVRVVInitUndef.cpp index ff7718c9ef6d..735fc1350c00 100644 --- a/llvm/lib/Target/RISCV/RISCVRVVInitUndef.cpp +++ b/llvm/lib/Target/RISCV/RISCVRVVInitUndef.cpp @@ -65,9 +65,7 @@ class RISCVInitUndef : public MachineFunctionPass { public: static char ID; - RISCVInitUndef() : MachineFunctionPass(ID) { - initializeRISCVInitUndefPass(*PassRegistry::getPassRegistry()); - } + RISCVInitUndef() : MachineFunctionPass(ID) {} bool runOnMachineFunction(MachineFunction &MF) override; void getAnalysisUsage(AnalysisUsage &AU) const override { diff --git a/llvm/lib/Target/RISCV/RISCVSubtarget.h b/llvm/lib/Target/RISCV/RISCVSubtarget.h index 52f00f1f0990..23d56cfa6e4e 100644 --- a/llvm/lib/Target/RISCV/RISCVSubtarget.h +++ b/llvm/lib/Target/RISCV/RISCVSubtarget.h @@ -25,6 +25,7 @@ #include "llvm/CodeGen/TargetSubtargetInfo.h" #include "llvm/IR/DataLayout.h" #include "llvm/Target/TargetMachine.h" +#include #define GET_SUBTARGETINFO_HEADER #include "RISCVGenSubtargetInfo.inc" @@ -192,7 +193,10 @@ public: return UserReservedRegister[i]; } - bool hasMacroFusion() const { return hasLUIADDIFusion(); } + bool hasMacroFusion() const { + return hasLUIADDIFusion() || hasAUIPCADDIFusion() || + hasShiftedZExtFusion() || hasLDADDFusion(); + } // Vector codegen related methods. bool hasVInstructions() const { return HasStdExtZve32x; } diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp index ba13f8d2f448..3abdb6003659 100644 --- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp +++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp @@ -213,13 +213,8 @@ RISCVTargetMachine::getSubtargetImpl(const Function &F) const { llvm::bit_floor((RVVBitsMax < 64 || RVVBitsMax > 65536) ? 0 : RVVBitsMax); SmallString<512> Key; - Key += "RVVMin"; - Key += std::to_string(RVVBitsMin); - Key += "RVVMax"; - Key += std::to_string(RVVBitsMax); - Key += CPU; - Key += TuneCPU; - Key += FS; + raw_svector_ostream(Key) << "RVVMin" << RVVBitsMin << "RVVMax" << RVVBitsMax + << CPU << TuneCPU << FS; auto &I = SubtargetMap[Key]; if (!I) { // This needs to be done before we create a new subtarget since any @@ -549,7 +544,7 @@ void RISCVPassConfig::addMachineSSAOptimization() { if (EnableMachineCombiner) addPass(&MachineCombinerID); - if (TM->getTargetTriple().getArch() == Triple::riscv64) { + if (TM->getTargetTriple().isRISCV64()) { addPass(createRISCVOptWInstrsPass()); } } diff --git a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp index 000d68d4a79f..ec62a819b00e 100644 --- a/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVEmitIntrinsics.cpp @@ -505,7 +505,6 @@ bool SPIRVEmitIntrinsics::runOnFunction(Function &Func) { if (!SI) continue; Type *ElTy = SI->getValueOperand()->getType(); - PointerType *PTy = cast(SI->getOperand(1)->getType()); if (ElTy->isAggregateType() || ElTy->isVectorTy()) AggrStores.insert(&I); } diff --git a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp index 0e3fcf4701e6..d450613196f3 100644 --- a/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVGlobalRegistry.cpp @@ -707,12 +707,7 @@ SPIRVType *SPIRVGlobalRegistry::createSPIRVType( // At the moment, all opaque pointers correspond to i8 element type. // TODO: change the implementation once opaque pointers are supported // in the SPIR-V specification. - if (PType->isOpaque()) - SpvElementType = getOrCreateSPIRVIntegerType(8, MIRBuilder); - else - SpvElementType = - findSPIRVType(PType->getNonOpaquePointerElementType(), MIRBuilder, - SPIRV::AccessQualifier::ReadWrite, EmitIR); + SpvElementType = getOrCreateSPIRVIntegerType(8, MIRBuilder); auto SC = addressSpaceToStorageClass(PType->getAddressSpace()); // Null pointer means we have a loop in type definitions, make and // return corresponding OpTypeForwardPointer. diff --git a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp index 6cb51d782919..0bd51436082d 100644 --- a/llvm/lib/Target/SPIRV/SPIRVUtils.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVUtils.cpp @@ -326,10 +326,9 @@ std::string getOclOrSpirvBuiltinDemangledName(StringRef Name) { } const Type *getTypedPtrEltType(const Type *Ty) { - auto PType = dyn_cast(Ty); - if (!PType || PType->isOpaque()) - return Ty; - return PType->getNonOpaquePointerElementType(); + // TODO: This function requires updating following the opaque pointer + // migration. + return Ty; } bool hasBuiltinTypePrefix(StringRef Name) { diff --git a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp index 9e14f96b6caa..240f5396855c 100644 --- a/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp +++ b/llvm/lib/Target/Sparc/MCTargetDesc/SparcAsmBackend.cpp @@ -272,7 +272,8 @@ namespace { } bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override { + const MCValue &Target, + const MCSubtargetInfo *STI) override { if (Fixup.getKind() >= FirstLiteralRelocationKind) return true; switch ((Sparc::Fixups)Fixup.getKind()) { diff --git a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp index eafe41124897..e32023ccedf5 100644 --- a/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp +++ b/llvm/lib/Target/SystemZ/MCTargetDesc/SystemZMCAsmBackend.cpp @@ -116,7 +116,8 @@ public: std::optional getFixupKind(StringRef Name) const override; const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, uint64_t Value, bool IsResolved, @@ -164,8 +165,9 @@ SystemZMCAsmBackend::getFixupKindInfo(MCFixupKind Kind) const { } bool SystemZMCAsmBackend::shouldForceRelocation(const MCAssembler &, - const MCFixup &Fixup, - const MCValue &) { + const MCFixup &Fixup, + const MCValue &, + const MCSubtargetInfo *STI) { return Fixup.getKind() >= FirstLiteralRelocationKind; } diff --git a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp index 4cc69530db01..0d8d5451b813 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp @@ -1558,6 +1558,9 @@ void SystemZDAGToDAGISel::Select(SDNode *Node) { break; } } + // Don't split an XOR with -1 as LCGR/AGHI is more compact. + if (Opcode == ISD::XOR && Op1->isAllOnes()) + break; if (!SystemZ::isImmLF(Val) && !SystemZ::isImmHF(Val)) { splitLargeImmediate(Opcode, Node, Node->getOperand(0), Val - uint32_t(Val), uint32_t(Val)); diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp index 873994c2e333..f79787d4baa4 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp @@ -872,13 +872,21 @@ bool SystemZTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const TargetLowering::AtomicExpansionKind SystemZTargetLowering::shouldExpandAtomicRMWInIR(AtomicRMWInst *RMW) const { - // TODO: expand them all here instead of in backend. - return (RMW->isFloatingPointOperation() || - RMW->getOperation() == AtomicRMWInst::UIncWrap || - RMW->getOperation() == AtomicRMWInst::UDecWrap || - RMW->getType()->isIntegerTy(128)) - ? AtomicExpansionKind::CmpXChg - : AtomicExpansionKind::None; + // Don't expand subword operations as they require special treatment. + if (RMW->getType()->isIntegerTy(8) || RMW->getType()->isIntegerTy(16)) + return AtomicExpansionKind::None; + + // Don't expand if there is a target instruction available. + if (Subtarget.hasInterlockedAccess1() && + (RMW->getType()->isIntegerTy(32) || RMW->getType()->isIntegerTy(64)) && + (RMW->getOperation() == AtomicRMWInst::BinOp::Add || + RMW->getOperation() == AtomicRMWInst::BinOp::Sub || + RMW->getOperation() == AtomicRMWInst::BinOp::And || + RMW->getOperation() == AtomicRMWInst::BinOp::Or || + RMW->getOperation() == AtomicRMWInst::BinOp::Xor)) + return AtomicExpansionKind::None; + + return AtomicExpansionKind::CmpXChg; } bool SystemZTargetLowering::isLegalICmpImmediate(int64_t Imm) const { @@ -4350,6 +4358,31 @@ SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op, return Chain; } +// Prepare for a Compare And Swap for a subword operation. This needs to be +// done in memory with 4 bytes at natural alignment. +static void getCSAddressAndShifts(SDValue Addr, SelectionDAG &DAG, SDLoc DL, + SDValue &AlignedAddr, SDValue &BitShift, + SDValue &NegBitShift) { + EVT PtrVT = Addr.getValueType(); + EVT WideVT = MVT::i32; + + // Get the address of the containing word. + AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, + DAG.getConstant(-4, DL, PtrVT)); + + // Get the number of bits that the word must be rotated left in order + // to bring the field to the top bits of a GR32. + BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, + DAG.getConstant(3, DL, PtrVT)); + BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); + + // Get the complementing shift amount, for rotating a field in the top + // bits back to its proper position. + NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, + DAG.getConstant(0, DL, WideVT), BitShift); + +} + // Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation. Lower the first // two into the fullword ATOMIC_LOADW_* operation given by Opcode. SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, @@ -4357,7 +4390,7 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, unsigned Opcode) const { auto *Node = cast(Op.getNode()); - // 32-bit operations need no code outside the main loop. + // 32-bit operations need no special handling. EVT NarrowVT = Node->getMemoryVT(); EVT WideVT = MVT::i32; if (NarrowVT == WideVT) @@ -4369,7 +4402,6 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, SDValue Src2 = Node->getVal(); MachineMemOperand *MMO = Node->getMemOperand(); SDLoc DL(Node); - EVT PtrVT = Addr.getValueType(); // Convert atomic subtracts of constants into additions. if (Opcode == SystemZISD::ATOMIC_LOADW_SUB) @@ -4378,20 +4410,8 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, Src2 = DAG.getConstant(-Const->getSExtValue(), DL, Src2.getValueType()); } - // Get the address of the containing word. - SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, - DAG.getConstant(-4, DL, PtrVT)); - - // Get the number of bits that the word must be rotated left in order - // to bring the field to the top bits of a GR32. - SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, - DAG.getConstant(3, DL, PtrVT)); - BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); - - // Get the complementing shift amount, for rotating a field in the top - // bits back to its proper position. - SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, - DAG.getConstant(0, DL, WideVT), BitShift); + SDValue AlignedAddr, BitShift, NegBitShift; + getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift); // Extend the source operand to 32 bits and prepare it for the inner loop. // ATOMIC_SWAPW uses RISBG to rotate the field left, but all other @@ -4423,38 +4443,24 @@ SDValue SystemZTargetLowering::lowerATOMIC_LOAD_OP(SDValue Op, return DAG.getMergeValues(RetOps, DL); } -// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations -// into ATOMIC_LOADW_SUBs and decide whether to convert 32- and 64-bit -// operations into additions. +// Op is an ATOMIC_LOAD_SUB operation. Lower 8- and 16-bit operations into +// ATOMIC_LOADW_SUBs and convert 32- and 64-bit operations into additions. SDValue SystemZTargetLowering::lowerATOMIC_LOAD_SUB(SDValue Op, SelectionDAG &DAG) const { auto *Node = cast(Op.getNode()); EVT MemVT = Node->getMemoryVT(); if (MemVT == MVT::i32 || MemVT == MVT::i64) { - // A full-width operation. + // A full-width operation: negate and use LAA(G). assert(Op.getValueType() == MemVT && "Mismatched VTs"); + assert(Subtarget.hasInterlockedAccess1() && + "Should have been expanded by AtomicExpand pass."); SDValue Src2 = Node->getVal(); - SDValue NegSrc2; SDLoc DL(Src2); - - if (auto *Op2 = dyn_cast(Src2)) { - // Use an addition if the operand is constant and either LAA(G) is - // available or the negative value is in the range of A(G)FHI. - int64_t Value = (-Op2->getAPIntValue()).getSExtValue(); - if (isInt<32>(Value) || Subtarget.hasInterlockedAccess1()) - NegSrc2 = DAG.getConstant(Value, DL, MemVT); - } else if (Subtarget.hasInterlockedAccess1()) - // Use LAA(G) if available. - NegSrc2 = DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT), - Src2); - - if (NegSrc2.getNode()) - return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT, - Node->getChain(), Node->getBasePtr(), NegSrc2, - Node->getMemOperand()); - - // Use the node as-is. - return Op; + SDValue NegSrc2 = + DAG.getNode(ISD::SUB, DL, MemVT, DAG.getConstant(0, DL, MemVT), Src2); + return DAG.getAtomic(ISD::ATOMIC_LOAD_ADD, DL, MemVT, + Node->getChain(), Node->getBasePtr(), NegSrc2, + Node->getMemOperand()); } return lowerATOMIC_LOAD_OP(Op, DAG, SystemZISD::ATOMIC_LOADW_SUB); @@ -4492,22 +4498,9 @@ SDValue SystemZTargetLowering::lowerATOMIC_CMP_SWAP(SDValue Op, // Convert 8-bit and 16-bit compare and swap to a loop, implemented // via a fullword ATOMIC_CMP_SWAPW operation. int64_t BitSize = NarrowVT.getSizeInBits(); - EVT PtrVT = Addr.getValueType(); - - // Get the address of the containing word. - SDValue AlignedAddr = DAG.getNode(ISD::AND, DL, PtrVT, Addr, - DAG.getConstant(-4, DL, PtrVT)); - // Get the number of bits that the word must be rotated left in order - // to bring the field to the top bits of a GR32. - SDValue BitShift = DAG.getNode(ISD::SHL, DL, PtrVT, Addr, - DAG.getConstant(3, DL, PtrVT)); - BitShift = DAG.getNode(ISD::TRUNCATE, DL, WideVT, BitShift); - - // Get the complementing shift amount, for rotating a field in the top - // bits back to its proper position. - SDValue NegBitShift = DAG.getNode(ISD::SUB, DL, WideVT, - DAG.getConstant(0, DL, WideVT), BitShift); + SDValue AlignedAddr, BitShift, NegBitShift; + getCSAddressAndShifts(Addr, DAG, DL, AlignedAddr, BitShift, NegBitShift); // Construct the ATOMIC_CMP_SWAPW node. SDVTList VTList = DAG.getVTList(WideVT, MVT::i32, MVT::Other); @@ -7951,20 +7944,17 @@ MachineBasicBlock *SystemZTargetLowering::emitCondStore(MachineInstr &MI, return JoinMBB; } -// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_LOAD{,W}_* -// or ATOMIC_SWAP{,W} instruction MI. BinOpcode is the instruction that -// performs the binary operation elided by "*", or 0 for ATOMIC_SWAP{,W}. -// BitSize is the width of the field in bits, or 0 if this is a partword -// ATOMIC_LOADW_* or ATOMIC_SWAPW instruction, in which case the bitsize -// is one of the operands. Invert says whether the field should be -// inverted after performing BinOpcode (e.g. for NAND). +// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_LOADW_* or +// ATOMIC_SWAPW instruction MI. BinOpcode is the instruction that performs +// the binary operation elided by "*", or 0 for ATOMIC_SWAPW. Invert says +// whether the field should be inverted after performing BinOpcode (e.g. for +// NAND). MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary( MachineInstr &MI, MachineBasicBlock *MBB, unsigned BinOpcode, - unsigned BitSize, bool Invert) const { + bool Invert) const { MachineFunction &MF = *MBB->getParent(); const SystemZInstrInfo *TII = Subtarget.getInstrInfo(); MachineRegisterInfo &MRI = MF.getRegInfo(); - bool IsSubWord = (BitSize < 32); // Extract the operands. Base can be a register or a frame index. // Src2 can be a register or immediate. @@ -7972,31 +7962,22 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary( MachineOperand Base = earlyUseOperand(MI.getOperand(1)); int64_t Disp = MI.getOperand(2).getImm(); MachineOperand Src2 = earlyUseOperand(MI.getOperand(3)); - Register BitShift = IsSubWord ? MI.getOperand(4).getReg() : Register(); - Register NegBitShift = IsSubWord ? MI.getOperand(5).getReg() : Register(); + Register BitShift = MI.getOperand(4).getReg(); + Register NegBitShift = MI.getOperand(5).getReg(); + unsigned BitSize = MI.getOperand(6).getImm(); DebugLoc DL = MI.getDebugLoc(); - if (IsSubWord) - BitSize = MI.getOperand(6).getImm(); - - // Subword operations use 32-bit registers. - const TargetRegisterClass *RC = (BitSize <= 32 ? - &SystemZ::GR32BitRegClass : - &SystemZ::GR64BitRegClass); - unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; - unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; // Get the right opcodes for the displacement. - LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); - CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); + unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); + unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); assert(LOpcode && CSOpcode && "Displacement out of range"); // Create virtual registers for temporary results. - Register OrigVal = MRI.createVirtualRegister(RC); - Register OldVal = MRI.createVirtualRegister(RC); - Register NewVal = (BinOpcode || IsSubWord ? - MRI.createVirtualRegister(RC) : Src2.getReg()); - Register RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); - Register RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); + Register OrigVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register OldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register NewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register RotatedOldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register RotatedNewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); // Insert a basic block for the main loop. MachineBasicBlock *StartMBB = MBB; @@ -8023,39 +8004,28 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary( BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) .addReg(OrigVal).addMBB(StartMBB) .addReg(Dest).addMBB(LoopMBB); - if (IsSubWord) - BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) - .addReg(OldVal).addReg(BitShift).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) + .addReg(OldVal).addReg(BitShift).addImm(0); if (Invert) { // Perform the operation normally and then invert every bit of the field. - Register Tmp = MRI.createVirtualRegister(RC); + Register Tmp = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); BuildMI(MBB, DL, TII->get(BinOpcode), Tmp).addReg(RotatedOldVal).add(Src2); - if (BitSize <= 32) - // XILF with the upper BitSize bits set. - BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) - .addReg(Tmp).addImm(-1U << (32 - BitSize)); - else { - // Use LCGR and add -1 to the result, which is more compact than - // an XILF, XILH pair. - Register Tmp2 = MRI.createVirtualRegister(RC); - BuildMI(MBB, DL, TII->get(SystemZ::LCGR), Tmp2).addReg(Tmp); - BuildMI(MBB, DL, TII->get(SystemZ::AGHI), RotatedNewVal) - .addReg(Tmp2).addImm(-1); - } + // XILF with the upper BitSize bits set. + BuildMI(MBB, DL, TII->get(SystemZ::XILF), RotatedNewVal) + .addReg(Tmp).addImm(-1U << (32 - BitSize)); } else if (BinOpcode) // A simply binary operation. BuildMI(MBB, DL, TII->get(BinOpcode), RotatedNewVal) .addReg(RotatedOldVal) .add(Src2); - else if (IsSubWord) + else // Use RISBG to rotate Src2 into position and use it to replace the // field in RotatedOldVal. BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedNewVal) .addReg(RotatedOldVal).addReg(Src2.getReg()) .addImm(32).addImm(31 + BitSize).addImm(32 - BitSize); - if (IsSubWord) - BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) - .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) + .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); BuildMI(MBB, DL, TII->get(CSOpcode), Dest) .addReg(OldVal) .addReg(NewVal) @@ -8070,50 +8040,40 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadBinary( return DoneMBB; } -// Implement EmitInstrWithCustomInserter for pseudo -// ATOMIC_LOAD{,W}_{,U}{MIN,MAX} instruction MI. CompareOpcode is the +// Implement EmitInstrWithCustomInserter for subword pseudo +// ATOMIC_LOADW_{,U}{MIN,MAX} instruction MI. CompareOpcode is the // instruction that should be used to compare the current field with the // minimum or maximum value. KeepOldMask is the BRC condition-code mask -// for when the current field should be kept. BitSize is the width of -// the field in bits, or 0 if this is a partword ATOMIC_LOADW_* instruction. +// for when the current field should be kept. MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax( MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode, - unsigned KeepOldMask, unsigned BitSize) const { + unsigned KeepOldMask) const { MachineFunction &MF = *MBB->getParent(); const SystemZInstrInfo *TII = Subtarget.getInstrInfo(); MachineRegisterInfo &MRI = MF.getRegInfo(); - bool IsSubWord = (BitSize < 32); // Extract the operands. Base can be a register or a frame index. Register Dest = MI.getOperand(0).getReg(); MachineOperand Base = earlyUseOperand(MI.getOperand(1)); int64_t Disp = MI.getOperand(2).getImm(); Register Src2 = MI.getOperand(3).getReg(); - Register BitShift = (IsSubWord ? MI.getOperand(4).getReg() : Register()); - Register NegBitShift = (IsSubWord ? MI.getOperand(5).getReg() : Register()); + Register BitShift = MI.getOperand(4).getReg(); + Register NegBitShift = MI.getOperand(5).getReg(); + unsigned BitSize = MI.getOperand(6).getImm(); DebugLoc DL = MI.getDebugLoc(); - if (IsSubWord) - BitSize = MI.getOperand(6).getImm(); - - // Subword operations use 32-bit registers. - const TargetRegisterClass *RC = (BitSize <= 32 ? - &SystemZ::GR32BitRegClass : - &SystemZ::GR64BitRegClass); - unsigned LOpcode = BitSize <= 32 ? SystemZ::L : SystemZ::LG; - unsigned CSOpcode = BitSize <= 32 ? SystemZ::CS : SystemZ::CSG; // Get the right opcodes for the displacement. - LOpcode = TII->getOpcodeForOffset(LOpcode, Disp); - CSOpcode = TII->getOpcodeForOffset(CSOpcode, Disp); + unsigned LOpcode = TII->getOpcodeForOffset(SystemZ::L, Disp); + unsigned CSOpcode = TII->getOpcodeForOffset(SystemZ::CS, Disp); assert(LOpcode && CSOpcode && "Displacement out of range"); // Create virtual registers for temporary results. - Register OrigVal = MRI.createVirtualRegister(RC); - Register OldVal = MRI.createVirtualRegister(RC); - Register NewVal = MRI.createVirtualRegister(RC); - Register RotatedOldVal = (IsSubWord ? MRI.createVirtualRegister(RC) : OldVal); - Register RotatedAltVal = (IsSubWord ? MRI.createVirtualRegister(RC) : Src2); - Register RotatedNewVal = (IsSubWord ? MRI.createVirtualRegister(RC) : NewVal); + Register OrigVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register OldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register NewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register RotatedOldVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register RotatedAltVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); + Register RotatedNewVal = MRI.createVirtualRegister(&SystemZ::GR32BitRegClass); // Insert 3 basic blocks for the loop. MachineBasicBlock *StartMBB = MBB; @@ -8139,9 +8099,8 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax( BuildMI(MBB, DL, TII->get(SystemZ::PHI), OldVal) .addReg(OrigVal).addMBB(StartMBB) .addReg(Dest).addMBB(UpdateMBB); - if (IsSubWord) - BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) - .addReg(OldVal).addReg(BitShift).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::RLL), RotatedOldVal) + .addReg(OldVal).addReg(BitShift).addImm(0); BuildMI(MBB, DL, TII->get(CompareOpcode)) .addReg(RotatedOldVal).addReg(Src2); BuildMI(MBB, DL, TII->get(SystemZ::BRC)) @@ -8153,10 +8112,9 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax( // %RotatedAltVal = RISBG %RotatedOldVal, %Src2, 32, 31 + BitSize, 0 // # fall through to UpdateMBB MBB = UseAltMBB; - if (IsSubWord) - BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) - .addReg(RotatedOldVal).addReg(Src2) - .addImm(32).addImm(31 + BitSize).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::RISBG32), RotatedAltVal) + .addReg(RotatedOldVal).addReg(Src2) + .addImm(32).addImm(31 + BitSize).addImm(0); MBB->addSuccessor(UpdateMBB); // UpdateMBB: @@ -8170,9 +8128,8 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax( BuildMI(MBB, DL, TII->get(SystemZ::PHI), RotatedNewVal) .addReg(RotatedOldVal).addMBB(LoopMBB) .addReg(RotatedAltVal).addMBB(UseAltMBB); - if (IsSubWord) - BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) - .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); + BuildMI(MBB, DL, TII->get(SystemZ::RLL), NewVal) + .addReg(RotatedNewVal).addReg(NegBitShift).addImm(0); BuildMI(MBB, DL, TII->get(CSOpcode), Dest) .addReg(OldVal) .addReg(NewVal) @@ -8187,7 +8144,7 @@ MachineBasicBlock *SystemZTargetLowering::emitAtomicLoadMinMax( return DoneMBB; } -// Implement EmitInstrWithCustomInserter for pseudo ATOMIC_CMP_SWAPW +// Implement EmitInstrWithCustomInserter for subword pseudo ATOMIC_CMP_SWAPW // instruction MI. MachineBasicBlock * SystemZTargetLowering::emitAtomicCmpSwapW(MachineInstr &MI, @@ -9004,171 +8961,44 @@ MachineBasicBlock *SystemZTargetLowering::EmitInstrWithCustomInserter( return emitExt128(MI, MBB, true); case SystemZ::ATOMIC_SWAPW: - return emitAtomicLoadBinary(MI, MBB, 0, 0); - case SystemZ::ATOMIC_SWAP_32: - return emitAtomicLoadBinary(MI, MBB, 0, 32); - case SystemZ::ATOMIC_SWAP_64: - return emitAtomicLoadBinary(MI, MBB, 0, 64); + return emitAtomicLoadBinary(MI, MBB, 0); case SystemZ::ATOMIC_LOADW_AR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 0); + return emitAtomicLoadBinary(MI, MBB, SystemZ::AR); case SystemZ::ATOMIC_LOADW_AFI: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 0); - case SystemZ::ATOMIC_LOAD_AR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AR, 32); - case SystemZ::ATOMIC_LOAD_AHI: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AHI, 32); - case SystemZ::ATOMIC_LOAD_AFI: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI, 32); - case SystemZ::ATOMIC_LOAD_AGR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AGR, 64); - case SystemZ::ATOMIC_LOAD_AGHI: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AGHI, 64); - case SystemZ::ATOMIC_LOAD_AGFI: - return emitAtomicLoadBinary(MI, MBB, SystemZ::AGFI, 64); + return emitAtomicLoadBinary(MI, MBB, SystemZ::AFI); case SystemZ::ATOMIC_LOADW_SR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 0); - case SystemZ::ATOMIC_LOAD_SR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::SR, 32); - case SystemZ::ATOMIC_LOAD_SGR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::SGR, 64); + return emitAtomicLoadBinary(MI, MBB, SystemZ::SR); case SystemZ::ATOMIC_LOADW_NR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0); + return emitAtomicLoadBinary(MI, MBB, SystemZ::NR); case SystemZ::ATOMIC_LOADW_NILH: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0); - case SystemZ::ATOMIC_LOAD_NR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32); - case SystemZ::ATOMIC_LOAD_NILL: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32); - case SystemZ::ATOMIC_LOAD_NILH: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32); - case SystemZ::ATOMIC_LOAD_NILF: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32); - case SystemZ::ATOMIC_LOAD_NGR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64); - case SystemZ::ATOMIC_LOAD_NILL64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64); - case SystemZ::ATOMIC_LOAD_NILH64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64); - case SystemZ::ATOMIC_LOAD_NIHL64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64); - case SystemZ::ATOMIC_LOAD_NIHH64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64); - case SystemZ::ATOMIC_LOAD_NILF64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64); - case SystemZ::ATOMIC_LOAD_NIHF64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64); + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH); case SystemZ::ATOMIC_LOADW_OR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 0); + return emitAtomicLoadBinary(MI, MBB, SystemZ::OR); case SystemZ::ATOMIC_LOADW_OILH: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 0); - case SystemZ::ATOMIC_LOAD_OR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OR, 32); - case SystemZ::ATOMIC_LOAD_OILL: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL, 32); - case SystemZ::ATOMIC_LOAD_OILH: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH, 32); - case SystemZ::ATOMIC_LOAD_OILF: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF, 32); - case SystemZ::ATOMIC_LOAD_OGR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OGR, 64); - case SystemZ::ATOMIC_LOAD_OILL64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILL64, 64); - case SystemZ::ATOMIC_LOAD_OILH64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH64, 64); - case SystemZ::ATOMIC_LOAD_OIHL64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHL64, 64); - case SystemZ::ATOMIC_LOAD_OIHH64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHH64, 64); - case SystemZ::ATOMIC_LOAD_OILF64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OILF64, 64); - case SystemZ::ATOMIC_LOAD_OIHF64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::OIHF64, 64); + return emitAtomicLoadBinary(MI, MBB, SystemZ::OILH); case SystemZ::ATOMIC_LOADW_XR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 0); + return emitAtomicLoadBinary(MI, MBB, SystemZ::XR); case SystemZ::ATOMIC_LOADW_XILF: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 0); - case SystemZ::ATOMIC_LOAD_XR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XR, 32); - case SystemZ::ATOMIC_LOAD_XILF: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF, 32); - case SystemZ::ATOMIC_LOAD_XGR: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XGR, 64); - case SystemZ::ATOMIC_LOAD_XILF64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF64, 64); - case SystemZ::ATOMIC_LOAD_XIHF64: - return emitAtomicLoadBinary(MI, MBB, SystemZ::XIHF64, 64); + return emitAtomicLoadBinary(MI, MBB, SystemZ::XILF); case SystemZ::ATOMIC_LOADW_NRi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 0, true); + return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, true); case SystemZ::ATOMIC_LOADW_NILHi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 0, true); - case SystemZ::ATOMIC_LOAD_NRi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NR, 32, true); - case SystemZ::ATOMIC_LOAD_NILLi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL, 32, true); - case SystemZ::ATOMIC_LOAD_NILHi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, 32, true); - case SystemZ::ATOMIC_LOAD_NILFi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF, 32, true); - case SystemZ::ATOMIC_LOAD_NGRi: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NGR, 64, true); - case SystemZ::ATOMIC_LOAD_NILL64i: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILL64, 64, true); - case SystemZ::ATOMIC_LOAD_NILH64i: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH64, 64, true); - case SystemZ::ATOMIC_LOAD_NIHL64i: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHL64, 64, true); - case SystemZ::ATOMIC_LOAD_NIHH64i: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHH64, 64, true); - case SystemZ::ATOMIC_LOAD_NILF64i: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NILF64, 64, true); - case SystemZ::ATOMIC_LOAD_NIHF64i: - return emitAtomicLoadBinary(MI, MBB, SystemZ::NIHF64, 64, true); + return emitAtomicLoadBinary(MI, MBB, SystemZ::NILH, true); case SystemZ::ATOMIC_LOADW_MIN: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, - SystemZ::CCMASK_CMP_LE, 0); - case SystemZ::ATOMIC_LOAD_MIN_32: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, - SystemZ::CCMASK_CMP_LE, 32); - case SystemZ::ATOMIC_LOAD_MIN_64: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, - SystemZ::CCMASK_CMP_LE, 64); - + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, SystemZ::CCMASK_CMP_LE); case SystemZ::ATOMIC_LOADW_MAX: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, - SystemZ::CCMASK_CMP_GE, 0); - case SystemZ::ATOMIC_LOAD_MAX_32: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, - SystemZ::CCMASK_CMP_GE, 32); - case SystemZ::ATOMIC_LOAD_MAX_64: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CGR, - SystemZ::CCMASK_CMP_GE, 64); - + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CR, SystemZ::CCMASK_CMP_GE); case SystemZ::ATOMIC_LOADW_UMIN: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, - SystemZ::CCMASK_CMP_LE, 0); - case SystemZ::ATOMIC_LOAD_UMIN_32: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, - SystemZ::CCMASK_CMP_LE, 32); - case SystemZ::ATOMIC_LOAD_UMIN_64: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, - SystemZ::CCMASK_CMP_LE, 64); - + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, SystemZ::CCMASK_CMP_LE); case SystemZ::ATOMIC_LOADW_UMAX: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, - SystemZ::CCMASK_CMP_GE, 0); - case SystemZ::ATOMIC_LOAD_UMAX_32: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, - SystemZ::CCMASK_CMP_GE, 32); - case SystemZ::ATOMIC_LOAD_UMAX_64: - return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLGR, - SystemZ::CCMASK_CMP_GE, 64); + return emitAtomicLoadMinMax(MI, MBB, SystemZ::CLR, SystemZ::CCMASK_CMP_GE); case SystemZ::ATOMIC_CMP_SWAPW: return emitAtomicCmpSwapW(MI, MBB); diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h index 40fe433f816f..1e2887cff816 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h +++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h @@ -748,13 +748,12 @@ private: bool ClearEven) const; MachineBasicBlock *emitAtomicLoadBinary(MachineInstr &MI, MachineBasicBlock *BB, - unsigned BinOpcode, unsigned BitSize, + unsigned BinOpcode, bool Invert = false) const; MachineBasicBlock *emitAtomicLoadMinMax(MachineInstr &MI, MachineBasicBlock *MBB, unsigned CompareOpcode, - unsigned KeepOldMask, - unsigned BitSize) const; + unsigned KeepOldMask) const; MachineBasicBlock *emitAtomicCmpSwapW(MachineInstr &MI, MachineBasicBlock *BB) const; MachineBasicBlock *emitMemMemWrapper(MachineInstr &MI, MachineBasicBlock *BB, diff --git a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td index a25719f80ad0..2e5ff4a1df67 100644 --- a/llvm/lib/Target/SystemZ/SystemZInstrFormats.td +++ b/llvm/lib/Target/SystemZ/SystemZInstrFormats.td @@ -5327,30 +5327,6 @@ multiclass CondStores - : Pseudo<(outs cls:$dst), (ins bdaddr20only:$ptr, operand:$src2), - [(set cls:$dst, (operator bdaddr20only:$ptr, pat))]> { - let Defs = [CC]; - let Has20BitOffset = 1; - let mayLoad = 1; - let mayStore = 1; - let usesCustomInserter = 1; - let hasNoSchedulingInfo = 1; -} - -// Specializations of AtomicLoadWBinary. -class AtomicLoadBinaryReg32 - : AtomicLoadBinary; -class AtomicLoadBinaryImm32 - : AtomicLoadBinary; -class AtomicLoadBinaryReg64 - : AtomicLoadBinary; -class AtomicLoadBinaryImm64 - : AtomicLoadBinary; - // OPERATOR is ATOMIC_SWAPW or an ATOMIC_LOADW_* operation. PAT and OPERAND // describe the second (non-memory) operand. class AtomicLoadWBinary; -def ATOMIC_SWAP_32 : AtomicLoadBinaryReg32; -def ATOMIC_SWAP_64 : AtomicLoadBinaryReg64; def ATOMIC_LOADW_AR : AtomicLoadWBinaryReg; def ATOMIC_LOADW_AFI : AtomicLoadWBinaryImm; -let Predicates = [FeatureNoInterlockedAccess1] in { - def ATOMIC_LOAD_AR : AtomicLoadBinaryReg32; - def ATOMIC_LOAD_AHI : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_AFI : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_AGR : AtomicLoadBinaryReg64; - def ATOMIC_LOAD_AGHI : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_AGFI : AtomicLoadBinaryImm64; -} def ATOMIC_LOADW_SR : AtomicLoadWBinaryReg; -def ATOMIC_LOAD_SR : AtomicLoadBinaryReg32; -def ATOMIC_LOAD_SGR : AtomicLoadBinaryReg64; def ATOMIC_LOADW_NR : AtomicLoadWBinaryReg; def ATOMIC_LOADW_NILH : AtomicLoadWBinaryImm; -let Predicates = [FeatureNoInterlockedAccess1] in { - def ATOMIC_LOAD_NR : AtomicLoadBinaryReg32; - def ATOMIC_LOAD_NILL : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_NILH : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_NILF : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_NGR : AtomicLoadBinaryReg64; - def ATOMIC_LOAD_NILL64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_NILH64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_NIHL64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_NIHH64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_NILF64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_NIHF64 : AtomicLoadBinaryImm64; -} def ATOMIC_LOADW_OR : AtomicLoadWBinaryReg; def ATOMIC_LOADW_OILH : AtomicLoadWBinaryImm; -let Predicates = [FeatureNoInterlockedAccess1] in { - def ATOMIC_LOAD_OR : AtomicLoadBinaryReg32; - def ATOMIC_LOAD_OILL : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_OILH : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_OILF : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_OGR : AtomicLoadBinaryReg64; - def ATOMIC_LOAD_OILL64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_OILH64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_OIHL64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_OIHH64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_OILF64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_OIHF64 : AtomicLoadBinaryImm64; -} def ATOMIC_LOADW_XR : AtomicLoadWBinaryReg; def ATOMIC_LOADW_XILF : AtomicLoadWBinaryImm; -let Predicates = [FeatureNoInterlockedAccess1] in { - def ATOMIC_LOAD_XR : AtomicLoadBinaryReg32; - def ATOMIC_LOAD_XILF : AtomicLoadBinaryImm32; - def ATOMIC_LOAD_XGR : AtomicLoadBinaryReg64; - def ATOMIC_LOAD_XILF64 : AtomicLoadBinaryImm64; - def ATOMIC_LOAD_XIHF64 : AtomicLoadBinaryImm64; -} def ATOMIC_LOADW_NRi : AtomicLoadWBinaryReg; def ATOMIC_LOADW_NILHi : AtomicLoadWBinaryImm; -def ATOMIC_LOAD_NRi : AtomicLoadBinaryReg32; -def ATOMIC_LOAD_NILLi : AtomicLoadBinaryImm32; -def ATOMIC_LOAD_NILHi : AtomicLoadBinaryImm32; -def ATOMIC_LOAD_NILFi : AtomicLoadBinaryImm32; -def ATOMIC_LOAD_NGRi : AtomicLoadBinaryReg64; -def ATOMIC_LOAD_NILL64i : AtomicLoadBinaryImm64; -def ATOMIC_LOAD_NILH64i : AtomicLoadBinaryImm64; -def ATOMIC_LOAD_NIHL64i : AtomicLoadBinaryImm64; -def ATOMIC_LOAD_NIHH64i : AtomicLoadBinaryImm64; -def ATOMIC_LOAD_NILF64i : AtomicLoadBinaryImm64; -def ATOMIC_LOAD_NIHF64i : AtomicLoadBinaryImm64; def ATOMIC_LOADW_MIN : AtomicLoadWBinaryReg; -def ATOMIC_LOAD_MIN_32 : AtomicLoadBinaryReg32; -def ATOMIC_LOAD_MIN_64 : AtomicLoadBinaryReg64; - def ATOMIC_LOADW_MAX : AtomicLoadWBinaryReg; -def ATOMIC_LOAD_MAX_32 : AtomicLoadBinaryReg32; -def ATOMIC_LOAD_MAX_64 : AtomicLoadBinaryReg64; - def ATOMIC_LOADW_UMIN : AtomicLoadWBinaryReg; -def ATOMIC_LOAD_UMIN_32 : AtomicLoadBinaryReg32; -def ATOMIC_LOAD_UMIN_64 : AtomicLoadBinaryReg64; - def ATOMIC_LOADW_UMAX : AtomicLoadWBinaryReg; -def ATOMIC_LOAD_UMAX_32 : AtomicLoadBinaryReg32; -def ATOMIC_LOAD_UMAX_64 : AtomicLoadBinaryReg64; def ATOMIC_CMP_SWAPW : Pseudo<(outs GR32:$dst), (ins bdaddr20only:$addr, GR32:$cmp, GR32:$swap, @@ -2346,6 +2263,10 @@ let isCodeGenOnly = 1, hasSideEffects = 1 in { def : Pat<(and (xor GR64:$x, (i64 -1)), GR64:$y), (XGR GR64:$y, (NGR GR64:$y, GR64:$x))>; +// Use LCGR/AGHI for i64 xor with -1. +def : Pat<(xor GR64:$x, (i64 -1)), + (AGHI (LCGR GR64:$x), (i64 -1))>; + // Shift/rotate instructions only use the last 6 bits of the second operand // register, so we can safely use NILL (16 fewer bits than NILF) to only AND the // last 16 bits. diff --git a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp index 2773a7aabab7..05f7c00bdc14 100644 --- a/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp +++ b/llvm/lib/Target/VE/MCTargetDesc/VEAsmBackend.cpp @@ -132,7 +132,8 @@ public: } bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override { + const MCValue &Target, + const MCSubtargetInfo *STI) override { switch ((VE::Fixups)Fixup.getKind()) { default: return false; diff --git a/llvm/lib/Target/X86/MCA/X86CustomBehaviour.h b/llvm/lib/Target/X86/MCA/X86CustomBehaviour.h index 3cb5edeee5b5..4a83ba848dd8 100644 --- a/llvm/lib/Target/X86/MCA/X86CustomBehaviour.h +++ b/llvm/lib/Target/X86/MCA/X86CustomBehaviour.h @@ -17,7 +17,6 @@ #ifndef LLVM_LIB_TARGET_X86_MCA_X86CUSTOMBEHAVIOUR_H #define LLVM_LIB_TARGET_X86_MCA_X86CUSTOMBEHAVIOUR_H -#include "llvm/ADT/SmallVector.h" #include "llvm/MCA/CustomBehaviour.h" #include "llvm/TargetParser/TargetParser.h" diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp index e01ce4f43143..99dc9797f6df 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86AsmBackend.cpp @@ -173,7 +173,8 @@ public: const MCFixupKindInfo &getFixupKindInfo(MCFixupKind Kind) const override; bool shouldForceRelocation(const MCAssembler &Asm, const MCFixup &Fixup, - const MCValue &Target) override; + const MCValue &Target, + const MCSubtargetInfo *STI) override; void applyFixup(const MCAssembler &Asm, const MCFixup &Fixup, const MCValue &Target, MutableArrayRef Data, @@ -645,8 +646,8 @@ const MCFixupKindInfo &X86AsmBackend::getFixupKindInfo(MCFixupKind Kind) const { } bool X86AsmBackend::shouldForceRelocation(const MCAssembler &, - const MCFixup &Fixup, - const MCValue &) { + const MCFixup &Fixup, const MCValue &, + const MCSubtargetInfo *STI) { return Fixup.getKind() >= FirstLiteralRelocationKind; } diff --git a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp index ee82faebb57e..20b37d5a9990 100644 --- a/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp +++ b/llvm/lib/Target/X86/MCTargetDesc/X86InstComments.cpp @@ -1285,8 +1285,8 @@ bool llvm::EmitAnyX86InstComments(const MCInst *MI, raw_ostream &OS, Src2Name = getRegName(MI->getOperand(2).getReg()); break; - case X86::VBROADCASTF128: - case X86::VBROADCASTI128: + case X86::VBROADCASTF128rm: + case X86::VBROADCASTI128rm: CASE_AVX512_INS_COMMON(BROADCASTF64X2, Z128, rm) CASE_AVX512_INS_COMMON(BROADCASTI64X2, Z128, rm) DecodeSubVectorBroadcast(4, 2, ShuffleMask); diff --git a/llvm/lib/Target/X86/X86FixupVectorConstants.cpp b/llvm/lib/Target/X86/X86FixupVectorConstants.cpp index 99e92bbcf996..5cc3b26dddaf 100644 --- a/llvm/lib/Target/X86/X86FixupVectorConstants.cpp +++ b/llvm/lib/Target/X86/X86FixupVectorConstants.cpp @@ -190,12 +190,13 @@ static Constant *rebuildSplatableConstant(const Constant *C, Type *SclTy = OriginalType->getScalarType(); unsigned NumSclBits = SclTy->getPrimitiveSizeInBits(); NumSclBits = std::min(NumSclBits, SplatBitWidth); + LLVMContext &Ctx = OriginalType->getContext(); if (NumSclBits == 8) { SmallVector RawBits; for (unsigned I = 0; I != SplatBitWidth; I += 8) RawBits.push_back(Splat->extractBits(8, I).getZExtValue()); - return ConstantDataVector::get(OriginalType->getContext(), RawBits); + return ConstantDataVector::get(Ctx, RawBits); } if (NumSclBits == 16) { @@ -204,25 +205,25 @@ static Constant *rebuildSplatableConstant(const Constant *C, RawBits.push_back(Splat->extractBits(16, I).getZExtValue()); if (SclTy->is16bitFPTy()) return ConstantDataVector::getFP(SclTy, RawBits); - return ConstantDataVector::get(OriginalType->getContext(), RawBits); + return ConstantDataVector::get(Ctx, RawBits); } if (NumSclBits == 32) { SmallVector RawBits; for (unsigned I = 0; I != SplatBitWidth; I += 32) RawBits.push_back(Splat->extractBits(32, I).getZExtValue()); - if (SclTy->isFloatTy()) - return ConstantDataVector::getFP(SclTy, RawBits); - return ConstantDataVector::get(OriginalType->getContext(), RawBits); + if (SclTy->isFloatingPointTy()) + return ConstantDataVector::getFP(Type::getFloatTy(Ctx), RawBits); + return ConstantDataVector::get(Ctx, RawBits); } // Fallback to i64 / double. SmallVector RawBits; for (unsigned I = 0; I != SplatBitWidth; I += 64) RawBits.push_back(Splat->extractBits(64, I).getZExtValue()); - if (SclTy->isDoubleTy()) - return ConstantDataVector::getFP(SclTy, RawBits); - return ConstantDataVector::get(OriginalType->getContext(), RawBits); + if (SclTy->isFloatingPointTy()) + return ConstantDataVector::getFP(Type::getDoubleTy(Ctx), RawBits); + return ConstantDataVector::get(Ctx, RawBits); } bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF, @@ -285,7 +286,7 @@ bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF, case X86::VMOVAPSYrm: case X86::VMOVUPDYrm: case X86::VMOVUPSYrm: - return ConvertToBroadcast(0, X86::VBROADCASTF128, X86::VBROADCASTSDYrm, + return ConvertToBroadcast(0, X86::VBROADCASTF128rm, X86::VBROADCASTSDYrm, X86::VBROADCASTSSYrm, 0, 0, 1); case X86::VMOVAPDZ128rm: case X86::VMOVAPSZ128rm: @@ -318,7 +319,7 @@ bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF, case X86::VMOVDQAYrm: case X86::VMOVDQUYrm: return ConvertToBroadcast( - 0, HasAVX2 ? X86::VBROADCASTI128 : X86::VBROADCASTF128, + 0, HasAVX2 ? X86::VBROADCASTI128rm : X86::VBROADCASTF128rm, HasAVX2 ? X86::VPBROADCASTQYrm : X86::VBROADCASTSDYrm, HasAVX2 ? X86::VPBROADCASTDYrm : X86::VBROADCASTSSYrm, HasAVX2 ? X86::VPBROADCASTWYrm : 0, HasAVX2 ? X86::VPBROADCASTBYrm : 0, diff --git a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp index d2e3c708f59d..b13bf361ab79 100644 --- a/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp +++ b/llvm/lib/Target/X86/X86FlagsCopyLowering.cpp @@ -24,7 +24,6 @@ #include "X86InstrBuilder.h" #include "X86InstrInfo.h" #include "X86Subtarget.h" -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/PostOrderIterator.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/ScopeExit.h" diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 4fca5afb46dd..adafb425babf 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -7040,6 +7040,31 @@ static SDValue combineToConsecutiveLoads(EVT VT, SDValue Op, const SDLoc &DL, IsAfterLegalize); } +static Constant *getConstantVector(MVT VT, ArrayRef Bits, + const APInt &Undefs, LLVMContext &C) { + unsigned ScalarSize = VT.getScalarSizeInBits(); + Type *Ty = EVT(VT.getScalarType()).getTypeForEVT(C); + + auto getConstantScalar = [&](const APInt &Val) -> Constant * { + if (VT.isFloatingPoint()) { + if (ScalarSize == 16) + return ConstantFP::get(C, APFloat(APFloat::IEEEhalf(), Val)); + if (ScalarSize == 32) + return ConstantFP::get(C, APFloat(APFloat::IEEEsingle(), Val)); + assert(ScalarSize == 64 && "Unsupported floating point scalar size"); + return ConstantFP::get(C, APFloat(APFloat::IEEEdouble(), Val)); + } + return Constant::getIntegerValue(Ty, Val); + }; + + SmallVector ConstantVec; + for (unsigned I = 0, E = Bits.size(); I != E; ++I) + ConstantVec.push_back(Undefs[I] ? UndefValue::get(Ty) + : getConstantScalar(Bits[I])); + + return ConstantVector::get(ArrayRef(ConstantVec)); +} + static Constant *getConstantVector(MVT VT, const APInt &SplatValue, unsigned SplatBitSize, LLVMContext &C) { unsigned ScalarSize = VT.getScalarSizeInBits(); @@ -8727,6 +8752,7 @@ X86TargetLowering::LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const { // constants. Insertion into a zero vector is handled as a special-case // somewhere below here. if (NumConstants == NumElems - 1 && NumNonZero != 1 && + FrozenUndefMask.isZero() && (isOperationLegalOrCustom(ISD::INSERT_VECTOR_ELT, VT) || isOperationLegalOrCustom(ISD::VECTOR_SHUFFLE, VT))) { // Create an all-constant vector. The variable element in the old @@ -48706,7 +48732,7 @@ static SDValue canonicalizeBitSelect(SDNode *N, SelectionDAG &DAG, if (useVPTERNLOG(Subtarget, VT)) { // Emit a VPTERNLOG node directly - 0xCA is the imm code for A?B:C. // VPTERNLOG is only available as vXi32/64-bit types. - MVT OpSVT = EltSizeInBits == 32 ? MVT::i32 : MVT::i64; + MVT OpSVT = EltSizeInBits <= 32 ? MVT::i32 : MVT::i64; MVT OpVT = MVT::getVectorVT(OpSVT, VT.getSizeInBits() / OpSVT.getSizeInBits()); SDValue A = DAG.getBitcast(OpVT, N0.getOperand(1)); @@ -49885,8 +49911,8 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, User->getValueSizeInBits(0).getFixedValue() > RegVT.getFixedSizeInBits()) { if (User->getOpcode() == X86ISD::SUBV_BROADCAST_LOAD && - cast(User)->getBasePtr() == Ptr && - cast(User)->getMemoryVT().getSizeInBits() == + cast(User)->getBasePtr() == Ptr && + cast(User)->getMemoryVT().getSizeInBits() == MemVT.getSizeInBits()) { SDValue Extract = extractSubVector(SDValue(User, 0), 0, DAG, SDLoc(N), RegVT.getSizeInBits()); @@ -49914,7 +49940,7 @@ static SDValue combineLoad(SDNode *N, SelectionDAG &DAG, if (ISD::isNormalLoad(User)) { // See if we are loading a constant that matches in the lower // bits of a longer constant (but from a different constant pool ptr). - SDValue UserPtr = cast(User)->getBasePtr(); + SDValue UserPtr = cast(User)->getBasePtr(); const Constant *LdC = getTargetConstantFromBasePtr(Ptr); const Constant *UserC = getTargetConstantFromBasePtr(UserPtr); if (LdC && UserC && UserPtr != Ptr && @@ -54481,6 +54507,8 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, SDValue Op0 = Ops[0]; bool IsSplat = llvm::all_equal(Ops); unsigned NumOps = Ops.size(); + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + LLVMContext &Ctx = *DAG.getContext(); // Repeated subvectors. if (IsSplat && @@ -54489,25 +54517,6 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, if (Op0.getOpcode() == X86ISD::VBROADCAST) return DAG.getNode(Op0.getOpcode(), DL, VT, Op0.getOperand(0)); - // If this simple subvector or scalar/subvector broadcast_load is inserted - // into both halves, use a larger broadcast_load. Update other uses to use - // an extracted subvector. - if (ISD::isNormalLoad(Op0.getNode()) || - Op0.getOpcode() == X86ISD::VBROADCAST_LOAD || - Op0.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) { - auto *Mem = cast(Op0); - unsigned Opc = Op0.getOpcode() == X86ISD::VBROADCAST_LOAD - ? X86ISD::VBROADCAST_LOAD - : X86ISD::SUBV_BROADCAST_LOAD; - if (SDValue BcastLd = - getBROADCAST_LOAD(Opc, DL, VT, Mem->getMemoryVT(), Mem, 0, DAG)) { - SDValue BcastSrc = - extractSubVector(BcastLd, 0, DAG, DL, Op0.getValueSizeInBits()); - DAG.ReplaceAllUsesOfValueWith(Op0, BcastSrc); - return BcastLd; - } - } - // concat_vectors(movddup(x),movddup(x)) -> broadcast(x) if (Op0.getOpcode() == X86ISD::MOVDDUP && VT == MVT::v4f64 && (Subtarget.hasAVX2() || @@ -54797,7 +54806,7 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, Subtarget.useAVX512Regs() && Subtarget.getPreferVectorWidth() >= 512 && (SrcVT.getScalarSizeInBits() > 16 || Subtarget.useBWIRegs())) { - EVT NewSrcVT = SrcVT.getDoubleNumVectorElementsVT(*DAG.getContext()); + EVT NewSrcVT = SrcVT.getDoubleNumVectorElementsVT(Ctx); return DAG.getNode(ISD::TRUNCATE, DL, VT, ConcatSubOperand(NewSrcVT, Ops, 0)); } @@ -54954,9 +54963,9 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, (EltSizeInBits >= 32 || Subtarget.hasBWI())) { EVT SelVT = Ops[0].getOperand(0).getValueType(); if (SelVT.getVectorElementType() == MVT::i1) { - SelVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, + SelVT = EVT::getVectorVT(Ctx, MVT::i1, NumOps * SelVT.getVectorNumElements()); - if (DAG.getTargetLoweringInfo().isTypeLegal(SelVT)) + if (TLI.isTypeLegal(SelVT)) return DAG.getNode(Op0.getOpcode(), DL, VT, ConcatSubOperand(SelVT.getSimpleVT(), Ops, 0), ConcatSubOperand(VT, Ops, 1), @@ -54969,8 +54978,8 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, (EltSizeInBits >= 32 || Subtarget.hasInt256()) && IsConcatFree(VT, Ops, 1) && IsConcatFree(VT, Ops, 2)) { EVT SelVT = Ops[0].getOperand(0).getValueType(); - SelVT = SelVT.getDoubleNumVectorElementsVT(*DAG.getContext()); - if (DAG.getTargetLoweringInfo().isTypeLegal(SelVT)) + SelVT = SelVT.getDoubleNumVectorElementsVT(Ctx); + if (TLI.isTypeLegal(SelVT)) return DAG.getNode(Op0.getOpcode(), DL, VT, ConcatSubOperand(SelVT.getSimpleVT(), Ops, 0), ConcatSubOperand(VT, Ops, 1), @@ -54985,7 +54994,7 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, if (auto *FirstLd = dyn_cast(peekThroughBitcasts(Op0))) { unsigned Fast; const X86TargetLowering *TLI = Subtarget.getTargetLowering(); - if (TLI->allowsMemoryAccess(*DAG.getContext(), DAG.getDataLayout(), VT, + if (TLI->allowsMemoryAccess(Ctx, DAG.getDataLayout(), VT, *FirstLd->getMemOperand(), &Fast) && Fast) { if (SDValue Ld = @@ -55002,13 +55011,44 @@ static SDValue combineConcatVectorOps(const SDLoc &DL, MVT VT, APInt OpUndefElts; SmallVector OpEltBits; if (!getTargetConstantBitsFromNode(Ops[I], EltSizeInBits, OpUndefElts, - OpEltBits, true, false)) - break; + OpEltBits, true, false)) + break; EltBits.append(OpEltBits); UndefElts.insertBits(OpUndefElts, I * OpUndefElts.getBitWidth()); } - if (EltBits.size() == VT.getVectorNumElements()) - return getConstVector(EltBits, UndefElts, VT, DAG, DL); + if (EltBits.size() == VT.getVectorNumElements()) { + Constant *C = getConstantVector(VT, EltBits, UndefElts, Ctx); + MVT PVT = TLI.getPointerTy(DAG.getDataLayout()); + SDValue CV = DAG.getConstantPool(C, PVT); + MachineFunction &MF = DAG.getMachineFunction(); + MachinePointerInfo MPI = MachinePointerInfo::getConstantPool(MF); + SDValue Ld = DAG.getLoad(VT, DL, DAG.getEntryNode(), CV, MPI); + SDValue Sub = extractSubVector(Ld, 0, DAG, DL, Op0.getValueSizeInBits()); + DAG.ReplaceAllUsesOfValueWith(Op0, Sub); + return Ld; + } + } + + // If this simple subvector or scalar/subvector broadcast_load is inserted + // into both halves, use a larger broadcast_load. Update other uses to use + // an extracted subvector. + if (IsSplat && + (VT.is256BitVector() || (VT.is512BitVector() && Subtarget.hasAVX512()))) { + if (ISD::isNormalLoad(Op0.getNode()) || + Op0.getOpcode() == X86ISD::VBROADCAST_LOAD || + Op0.getOpcode() == X86ISD::SUBV_BROADCAST_LOAD) { + auto *Mem = cast(Op0); + unsigned Opc = Op0.getOpcode() == X86ISD::VBROADCAST_LOAD + ? X86ISD::VBROADCAST_LOAD + : X86ISD::SUBV_BROADCAST_LOAD; + if (SDValue BcastLd = + getBROADCAST_LOAD(Opc, DL, VT, Mem->getMemoryVT(), Mem, 0, DAG)) { + SDValue BcastSrc = + extractSubVector(BcastLd, 0, DAG, DL, Op0.getValueSizeInBits()); + DAG.ReplaceAllUsesOfValueWith(Op0, BcastSrc); + return BcastLd; + } + } } // If we're splatting a 128-bit subvector to 512-bits, use SHUF128 directly. diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index a69ce40982ad..d6ae4971d238 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -8717,11 +8717,6 @@ bool X86InstrInfo::isSafeToMoveRegClassDefs( /// TODO: Eliminate this and move the code to X86MachineFunctionInfo. /// unsigned X86InstrInfo::getGlobalBaseReg(MachineFunction *MF) const { - assert((!Subtarget.is64Bit() || - MF->getTarget().getCodeModel() == CodeModel::Medium || - MF->getTarget().getCodeModel() == CodeModel::Large) && - "X86-64 PIC uses RIP relative addressing"); - X86MachineFunctionInfo *X86FI = MF->getInfo(); Register GlobalBaseReg = X86FI->getGlobalBaseReg(); if (GlobalBaseReg != 0) @@ -10083,12 +10078,6 @@ struct CGBR : public MachineFunctionPass { static_cast(&MF.getTarget()); const X86Subtarget &STI = MF.getSubtarget(); - // Don't do anything in the 64-bit small and kernel code models. They use - // RIP-relative addressing for everything. - if (STI.is64Bit() && (TM->getCodeModel() == CodeModel::Small || - TM->getCodeModel() == CodeModel::Kernel)) - return false; - // Only emit a global base reg in PIC mode. if (!TM->isPositionIndependent()) return false; @@ -10114,16 +10103,7 @@ struct CGBR : public MachineFunctionPass { PC = GlobalBaseReg; if (STI.is64Bit()) { - if (TM->getCodeModel() == CodeModel::Medium) { - // In the medium code model, use a RIP-relative LEA to materialize the - // GOT. - BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PC) - .addReg(X86::RIP) - .addImm(0) - .addReg(0) - .addExternalSymbol("_GLOBAL_OFFSET_TABLE_") - .addReg(0); - } else if (TM->getCodeModel() == CodeModel::Large) { + if (TM->getCodeModel() == CodeModel::Large) { // In the large code model, we are aiming for this code, though the // register allocation may vary: // leaq .LN$pb(%rip), %rax @@ -10146,7 +10126,14 @@ struct CGBR : public MachineFunctionPass { .addReg(PBReg, RegState::Kill) .addReg(GOTReg, RegState::Kill); } else { - llvm_unreachable("unexpected code model"); + // In other code models, use a RIP-relative LEA to materialize the + // GOT. + BuildMI(FirstMBB, MBBI, DL, TII->get(X86::LEA64r), PC) + .addReg(X86::RIP) + .addImm(0) + .addReg(0) + .addExternalSymbol("_GLOBAL_OFFSET_TABLE_") + .addReg(0); } } else { // Operand of MovePCtoStack is completely ignored by asm printer. It's diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td index 34eb17af1033..cf57fe562ed5 100644 --- a/llvm/lib/Target/X86/X86InstrSSE.td +++ b/llvm/lib/Target/X86/X86InstrSSE.td @@ -7093,35 +7093,35 @@ def VBROADCASTSDYrr : avx2_broadcast_rr<0x19, "vbroadcastsd", VR256, // halves of a 256-bit vector. // let mayLoad = 1, hasSideEffects = 0, Predicates = [HasAVX2] in -def VBROADCASTI128 : AVX8I<0x5A, MRMSrcMem, (outs VR256:$dst), - (ins i128mem:$src), - "vbroadcasti128\t{$src, $dst|$dst, $src}", []>, - Sched<[WriteShuffleLd]>, VEX, VEX_L; +def VBROADCASTI128rm : AVX8I<0x5A, MRMSrcMem, (outs VR256:$dst), + (ins i128mem:$src), + "vbroadcasti128\t{$src, $dst|$dst, $src}", []>, + Sched<[WriteShuffleLd]>, VEX, VEX_L; let mayLoad = 1, hasSideEffects = 0, Predicates = [HasAVX], ExeDomain = SSEPackedSingle in -def VBROADCASTF128 : AVX8I<0x1A, MRMSrcMem, (outs VR256:$dst), - (ins f128mem:$src), - "vbroadcastf128\t{$src, $dst|$dst, $src}", []>, - Sched<[SchedWriteFShuffle.XMM.Folded]>, VEX, VEX_L; +def VBROADCASTF128rm : AVX8I<0x1A, MRMSrcMem, (outs VR256:$dst), + (ins f128mem:$src), + "vbroadcastf128\t{$src, $dst|$dst, $src}", []>, + Sched<[SchedWriteFShuffle.XMM.Folded]>, VEX, VEX_L; let Predicates = [HasAVX, NoVLX] in { def : Pat<(v4f64 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; def : Pat<(v8f32 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; // NOTE: We're using FP instructions here, but execution domain fixing can // convert to integer when profitable. def : Pat<(v4i64 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; def : Pat<(v8i32 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; def : Pat<(v16i16 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; def : Pat<(v16f16 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; def : Pat<(v32i8 (X86SubVBroadcastld128 addr:$src)), - (VBROADCASTF128 addr:$src)>; + (VBROADCASTF128rm addr:$src)>; } //===----------------------------------------------------------------------===// diff --git a/llvm/lib/Target/X86/X86InstrSystem.td b/llvm/lib/Target/X86/X86InstrSystem.td index b55956169ff2..cbb5d4ed5bbd 100644 --- a/llvm/lib/Target/X86/X86InstrSystem.td +++ b/llvm/lib/Target/X86/X86InstrSystem.td @@ -679,6 +679,10 @@ def INVPCID32 : I<0x82, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), def INVPCID64 : I<0x82, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invpcid\t{$src2, $src1|$src1, $src2}", []>, T8PD, Requires<[In64BitMode, HasINVPCID]>; + +def INVPCID64_EVEX : I<0xF2, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), + "invpcid\t{$src2, $src1|$src1, $src2}", []>, + EVEX_NoCD8, T_MAP4XS, Requires<[In64BitMode, HasINVPCID]>; } // SchedRW let Predicates = [In64BitMode, HasINVPCID] in { diff --git a/llvm/lib/Target/X86/X86InstrUtils.td b/llvm/lib/Target/X86/X86InstrUtils.td index 071c41da6722..2f056f2ead62 100644 --- a/llvm/lib/Target/X86/X86InstrUtils.td +++ b/llvm/lib/Target/X86/X86InstrUtils.td @@ -86,6 +86,7 @@ class EVEX_CD8 { int CD8_EltSize = !srl(esize, 3); bits<3> CD8_Form = form.Value; } +class EVEX_NoCD8 : EVEX { bits<7> CD8_Scale = 0; } class XOP { Encoding OpEnc = EncXOP; } class XOP_4V : XOP { bit hasVEX_4V = 1; } class EVEX2VEXOverride { diff --git a/llvm/lib/Target/X86/X86InstrVMX.td b/llvm/lib/Target/X86/X86InstrVMX.td index c601b1a97932..5289819119ce 100644 --- a/llvm/lib/Target/X86/X86InstrVMX.td +++ b/llvm/lib/Target/X86/X86InstrVMX.td @@ -22,11 +22,9 @@ def INVEPT32 : I<0x80, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), def INVEPT64 : I<0x80, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invept\t{$src2, $src1|$src1, $src2}", []>, T8PD, Requires<[In64BitMode]>; - -let CD8_Scale = 0 in def INVEPT64_EVEX : I<0xF0, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invept\t{$src2, $src1|$src1, $src2}", []>, - EVEX, T_MAP4XS, Requires<[In64BitMode]>; + EVEX_NoCD8, T_MAP4XS, Requires<[In64BitMode]>; // 66 0F 38 81 def INVVPID32 : I<0x81, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), @@ -35,11 +33,9 @@ def INVVPID32 : I<0x81, MRMSrcMem, (outs), (ins GR32:$src1, i128mem:$src2), def INVVPID64 : I<0x81, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invvpid\t{$src2, $src1|$src1, $src2}", []>, T8PD, Requires<[In64BitMode]>; - -let CD8_Scale = 0 in def INVVPID64_EVEX : I<0xF1, MRMSrcMem, (outs), (ins GR64:$src1, i128mem:$src2), "invvpid\t{$src2, $src1|$src1, $src2}", []>, - EVEX, T_MAP4XS, Requires<[In64BitMode]>; + EVEX_NoCD8, T_MAP4XS, Requires<[In64BitMode]>; // 0F 01 C1 def VMCALL : I<0x01, MRM_C1, (outs), (ins), "vmcall", []>, TB; diff --git a/llvm/lib/Target/X86/X86MCInstLower.cpp b/llvm/lib/Target/X86/X86MCInstLower.cpp index cbb012161524..e1a67f61e766 100644 --- a/llvm/lib/Target/X86/X86MCInstLower.cpp +++ b/llvm/lib/Target/X86/X86MCInstLower.cpp @@ -1865,8 +1865,8 @@ static void addConstantComments(const MachineInstr *MI, // For loads from a constant pool to a vector register, print the constant // loaded. CASE_ALL_MOV_RM() - case X86::VBROADCASTF128: - case X86::VBROADCASTI128: + case X86::VBROADCASTF128rm: + case X86::VBROADCASTI128rm: case X86::VBROADCASTF32X4Z256rm: case X86::VBROADCASTF32X4rm: case X86::VBROADCASTF32X8rm: @@ -1891,8 +1891,8 @@ static void addConstantComments(const MachineInstr *MI, CASE_128_MOV_RM() NumLanes = 1; BitWidth = 128; break; CASE_256_MOV_RM() NumLanes = 1; BitWidth = 256; break; CASE_512_MOV_RM() NumLanes = 1; BitWidth = 512; break; - case X86::VBROADCASTF128: NumLanes = 2; BitWidth = 128; break; - case X86::VBROADCASTI128: NumLanes = 2; BitWidth = 128; break; + case X86::VBROADCASTF128rm: NumLanes = 2; BitWidth = 128; break; + case X86::VBROADCASTI128rm: NumLanes = 2; BitWidth = 128; break; case X86::VBROADCASTF32X4Z256rm: NumLanes = 2; BitWidth = 128; break; case X86::VBROADCASTF32X4rm: NumLanes = 4; BitWidth = 128; break; case X86::VBROADCASTF32X8rm: NumLanes = 2; BitWidth = 256; break; diff --git a/llvm/lib/Target/X86/X86ReplaceableInstrs.def b/llvm/lib/Target/X86/X86ReplaceableInstrs.def index 4798275c0519..e1383198d3fe 100644 --- a/llvm/lib/Target/X86/X86ReplaceableInstrs.def +++ b/llvm/lib/Target/X86/X86ReplaceableInstrs.def @@ -202,7 +202,7 @@ ENTRY(VBROADCASTSSYrr, VBROADCASTSSYrr, VPBROADCASTDYrr) ENTRY(VBROADCASTSSYrm, VBROADCASTSSYrm, VPBROADCASTDYrm) ENTRY(VBROADCASTSDYrr, VBROADCASTSDYrr, VPBROADCASTQYrr) ENTRY(VBROADCASTSDYrm, VBROADCASTSDYrm, VPBROADCASTQYrm) -ENTRY(VBROADCASTF128, VBROADCASTF128, VBROADCASTI128) +ENTRY(VBROADCASTF128rm, VBROADCASTF128rm, VBROADCASTI128rm) ENTRY(VBLENDPSYrri, VBLENDPSYrri, VPBLENDDYrri) ENTRY(VBLENDPSYrmi, VBLENDPSYrmi, VPBLENDDYrmi) ENTRY(VPERMILPSYmi, VPERMILPSYmi, VPSHUFDYmi) diff --git a/llvm/lib/Target/X86/X86SchedAlderlakeP.td b/llvm/lib/Target/X86/X86SchedAlderlakeP.td index 3406a28be2c2..8e3e55428264 100644 --- a/llvm/lib/Target/X86/X86SchedAlderlakeP.td +++ b/llvm/lib/Target/X86/X86SchedAlderlakeP.td @@ -1328,7 +1328,7 @@ def ADLPWriteResGroup117 : SchedWriteRes<[ADLPPort02_03_11]> { let Latency = 8; } def : InstRW<[ADLPWriteResGroup117], (instregex "^MMX_MOV(D|Q)64rm$", - "^VBROADCAST(F|I)128$", + "^VBROADCAST(F|I)128rm$", "^VBROADCASTS(D|S)Yrm$", "^VMOV(D|SH|SL)DUPYrm$", "^VPBROADCAST(D|Q)Yrm$")>; diff --git a/llvm/lib/Target/X86/X86SchedBroadwell.td b/llvm/lib/Target/X86/X86SchedBroadwell.td index 8575f747b4d3..61a8832000e2 100644 --- a/llvm/lib/Target/X86/X86SchedBroadwell.td +++ b/llvm/lib/Target/X86/X86SchedBroadwell.td @@ -946,8 +946,8 @@ def BWWriteResGroup58 : SchedWriteRes<[BWPort23]> { let ReleaseAtCycles = [1]; } def: InstRW<[BWWriteResGroup58], (instregex "LD_F(32|64|80)m")>; -def: InstRW<[BWWriteResGroup58], (instrs VBROADCASTF128, - VBROADCASTI128, +def: InstRW<[BWWriteResGroup58], (instrs VBROADCASTF128rm, + VBROADCASTI128rm, VBROADCASTSDYrm, VBROADCASTSSYrm, VMOVDDUPYrm, diff --git a/llvm/lib/Target/X86/X86SchedHaswell.td b/llvm/lib/Target/X86/X86SchedHaswell.td index d10d7684ac12..8795ca95c559 100644 --- a/llvm/lib/Target/X86/X86SchedHaswell.td +++ b/llvm/lib/Target/X86/X86SchedHaswell.td @@ -876,8 +876,8 @@ def HWWriteResGroup0_1 : SchedWriteRes<[HWPort23]> { let NumMicroOps = 1; let ReleaseAtCycles = [1]; } -def: InstRW<[HWWriteResGroup0_1], (instrs VBROADCASTF128, - VBROADCASTI128, +def: InstRW<[HWWriteResGroup0_1], (instrs VBROADCASTF128rm, + VBROADCASTI128rm, VBROADCASTSDYrm, VBROADCASTSSYrm, VMOVDDUPYrm, diff --git a/llvm/lib/Target/X86/X86SchedIceLake.td b/llvm/lib/Target/X86/X86SchedIceLake.td index a2aa2655bca2..e27af1433d45 100644 --- a/llvm/lib/Target/X86/X86SchedIceLake.td +++ b/llvm/lib/Target/X86/X86SchedIceLake.td @@ -1274,8 +1274,8 @@ def ICXWriteResGroup89 : SchedWriteRes<[ICXPort23]> { let ReleaseAtCycles = [1]; } def: InstRW<[ICXWriteResGroup89], (instregex "LD_F(32|64|80)m")>; -def: InstRW<[ICXWriteResGroup89], (instrs VBROADCASTF128, - VBROADCASTI128, +def: InstRW<[ICXWriteResGroup89], (instrs VBROADCASTF128rm, + VBROADCASTI128rm, VBROADCASTSDYrm, VBROADCASTSSYrm, VMOVDDUPYrm, diff --git a/llvm/lib/Target/X86/X86SchedSapphireRapids.td b/llvm/lib/Target/X86/X86SchedSapphireRapids.td index 6a426ef4cf54..4eac53385ae5 100644 --- a/llvm/lib/Target/X86/X86SchedSapphireRapids.td +++ b/llvm/lib/Target/X86/X86SchedSapphireRapids.td @@ -1599,7 +1599,7 @@ def SPRWriteResGroup126 : SchedWriteRes<[SPRPort02_03_11]> { let Latency = 8; } def : InstRW<[SPRWriteResGroup126], (instregex "^MMX_MOV(D|Q)64rm$", - "^VBROADCAST(F|I)128$", + "^VBROADCAST(F|I)128rm$", "^VBROADCAST(F|I)32X(2|4)Z256rm$", "^VBROADCAST(F|I)32X(8|2Z)rm$", "^VBROADCAST(F|I)(32|64)X4rm$", diff --git a/llvm/lib/Target/X86/X86SchedSkylakeClient.td b/llvm/lib/Target/X86/X86SchedSkylakeClient.td index 92ed491bc296..4fa138f69fb9 100644 --- a/llvm/lib/Target/X86/X86SchedSkylakeClient.td +++ b/llvm/lib/Target/X86/X86SchedSkylakeClient.td @@ -1064,8 +1064,8 @@ def SKLWriteResGroup85 : SchedWriteRes<[SKLPort23]> { let ReleaseAtCycles = [1]; } def: InstRW<[SKLWriteResGroup85], (instregex "LD_F(32|64|80)m")>; -def: InstRW<[SKLWriteResGroup85], (instrs VBROADCASTF128, - VBROADCASTI128, +def: InstRW<[SKLWriteResGroup85], (instrs VBROADCASTF128rm, + VBROADCASTI128rm, VBROADCASTSDYrm, VBROADCASTSSYrm, VMOVDDUPYrm, diff --git a/llvm/lib/Target/X86/X86SchedSkylakeServer.td b/llvm/lib/Target/X86/X86SchedSkylakeServer.td index ed22f95c83e5..8194af8a6e1d 100644 --- a/llvm/lib/Target/X86/X86SchedSkylakeServer.td +++ b/llvm/lib/Target/X86/X86SchedSkylakeServer.td @@ -1254,8 +1254,8 @@ def SKXWriteResGroup89 : SchedWriteRes<[SKXPort23]> { let ReleaseAtCycles = [1]; } def: InstRW<[SKXWriteResGroup89], (instregex "LD_F(32|64|80)m")>; -def: InstRW<[SKXWriteResGroup89], (instrs VBROADCASTF128, - VBROADCASTI128, +def: InstRW<[SKXWriteResGroup89], (instrs VBROADCASTF128rm, + VBROADCASTI128rm, VBROADCASTSDYrm, VBROADCASTSSYrm, VMOVDDUPYrm, diff --git a/llvm/lib/Target/X86/X86ScheduleBdVer2.td b/llvm/lib/Target/X86/X86ScheduleBdVer2.td index aeeabba45b65..c9749979576f 100644 --- a/llvm/lib/Target/X86/X86ScheduleBdVer2.td +++ b/llvm/lib/Target/X86/X86ScheduleBdVer2.td @@ -933,7 +933,7 @@ def PdWriteVBROADCASTF128 : SchedWriteRes<[PdFPU01, PdFPFMA]> { let ReleaseAtCycles = [1, 3]; let NumMicroOps = 2; } -def : InstRW<[PdWriteVBROADCASTF128], (instrs VBROADCASTF128)>; +def : InstRW<[PdWriteVBROADCASTF128], (instrs VBROADCASTF128rm)>; defm : PdWriteResXMMPair; defm : PdWriteResYMMPair; diff --git a/llvm/lib/Target/X86/X86ScheduleBtVer2.td b/llvm/lib/Target/X86/X86ScheduleBtVer2.td index 8b7d501981b1..9cba933e82b0 100644 --- a/llvm/lib/Target/X86/X86ScheduleBtVer2.td +++ b/llvm/lib/Target/X86/X86ScheduleBtVer2.td @@ -816,7 +816,7 @@ def JWriteVBROADCASTYLd: SchedWriteRes<[JLAGU, JFPU01, JFPX]> { } def : InstRW<[JWriteVBROADCASTYLd], (instrs VBROADCASTSDYrm, VBROADCASTSSYrm, - VBROADCASTF128)>; + VBROADCASTF128rm)>; def JWriteJVZEROALL: SchedWriteRes<[]> { let Latency = 90; diff --git a/llvm/lib/Target/X86/X86ScheduleZnver1.td b/llvm/lib/Target/X86/X86ScheduleZnver1.td index e39bcf807a0c..7ee9eadf8439 100644 --- a/llvm/lib/Target/X86/X86ScheduleZnver1.td +++ b/llvm/lib/Target/X86/X86ScheduleZnver1.td @@ -996,8 +996,8 @@ def ZnWriteBROADCAST : SchedWriteRes<[ZnAGU, ZnFPU13]> { let Latency = 8; } // VBROADCASTF128 / VBROADCASTI128. -def : InstRW<[ZnWriteBROADCAST], (instrs VBROADCASTF128, - VBROADCASTI128)>; +def : InstRW<[ZnWriteBROADCAST], (instrs VBROADCASTF128rm, + VBROADCASTI128rm)>; // EXTRACTPS. // r32,x,i. diff --git a/llvm/lib/Target/X86/X86ScheduleZnver2.td b/llvm/lib/Target/X86/X86ScheduleZnver2.td index ecaca70bb677..c0775847798d 100644 --- a/llvm/lib/Target/X86/X86ScheduleZnver2.td +++ b/llvm/lib/Target/X86/X86ScheduleZnver2.td @@ -1004,8 +1004,8 @@ def Zn2WriteBROADCAST : SchedWriteRes<[Zn2AGU, Zn2FPU13]> { let Latency = 8; } // VBROADCASTF128 / VBROADCASTI128. -def : InstRW<[Zn2WriteBROADCAST], (instrs VBROADCASTF128, - VBROADCASTI128)>; +def : InstRW<[Zn2WriteBROADCAST], (instrs VBROADCASTF128rm, + VBROADCASTI128rm)>; // EXTRACTPS. // r32,x,i. diff --git a/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp b/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp index 2a896314875d..6301285fe954 100644 --- a/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp +++ b/llvm/lib/Target/X86/X86SpeculativeLoadHardening.cpp @@ -26,7 +26,6 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/SmallPtrSet.h" #include "llvm/ADT/SmallSet.h" #include "llvm/ADT/SmallVector.h" diff --git a/llvm/lib/TargetParser/Host.cpp b/llvm/lib/TargetParser/Host.cpp index ae47929fce6a..40f4ba0be550 100644 --- a/llvm/lib/TargetParser/Host.cpp +++ b/llvm/lib/TargetParser/Host.cpp @@ -202,6 +202,7 @@ StringRef sys::detail::getHostCPUNameForARM(StringRef ProcCpuinfoContent) { .Case("0xc20", "cortex-m0") .Case("0xc23", "cortex-m3") .Case("0xc24", "cortex-m4") + .Case("0xd24", "cortex-m52") .Case("0xd22", "cortex-m55") .Case("0xd02", "cortex-a34") .Case("0xd04", "cortex-a35") diff --git a/llvm/lib/Transforms/Hello/CMakeLists.txt b/llvm/lib/Transforms/Hello/CMakeLists.txt index c4f10247c1a6..9510c31f633f 100644 --- a/llvm/lib/Transforms/Hello/CMakeLists.txt +++ b/llvm/lib/Transforms/Hello/CMakeLists.txt @@ -6,7 +6,7 @@ if( NOT LLVM_REQUIRES_RTTI ) endif() endif() -if(WIN32 OR CYGWIN) +if(WIN32 OR CYGWIN OR ZOS) set(LLVM_LINK_COMPONENTS Core Support) endif() diff --git a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp index 889ebd7438bd..cbe0a96976c3 100644 --- a/llvm/lib/Transforms/IPO/AttributorAttributes.cpp +++ b/llvm/lib/Transforms/IPO/AttributorAttributes.cpp @@ -290,20 +290,19 @@ static const Value *getPointerOperand(const Instruction *I, return nullptr; } -/// Helper function to create a pointer of type \p ResTy, based on \p Ptr, and -/// advanced by \p Offset bytes. To aid later analysis the method tries to build +/// Helper function to create a pointer based on \p Ptr, and advanced by \p +/// Offset bytes. To aid later analysis the method tries to build /// getelement pointer instructions that traverse the natural type of \p Ptr if /// possible. If that fails, the remaining offset is adjusted byte-wise, hence /// through a cast to i8*. /// /// TODO: This could probably live somewhere more prominantly if it doesn't /// already exist. -static Value *constructPointer(Type *ResTy, Type *PtrElemTy, Value *Ptr, - int64_t Offset, IRBuilder &IRB, - const DataLayout &DL) { +static Value *constructPointer(Type *PtrElemTy, Value *Ptr, int64_t Offset, + IRBuilder &IRB, const DataLayout &DL) { assert(Offset >= 0 && "Negative offset not supported yet!"); LLVM_DEBUG(dbgs() << "Construct pointer: " << *Ptr << " + " << Offset - << "-bytes as " << *ResTy << "\n"); + << "-bytes\n"); if (Offset) { Type *Ty = PtrElemTy; @@ -327,10 +326,6 @@ static Value *constructPointer(Type *ResTy, Type *PtrElemTy, Value *Ptr, } } - // Ensure the result has the requested type. - Ptr = IRB.CreatePointerBitCastOrAddrSpaceCast(Ptr, ResTy, - Ptr->getName() + ".cast"); - LLVM_DEBUG(dbgs() << "Constructed pointer: " << *Ptr << "\n"); return Ptr; } @@ -7492,19 +7487,16 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { if (auto *PrivStructType = dyn_cast(PrivType)) { const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType); for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) { - Type *PointeeTy = PrivStructType->getElementType(u)->getPointerTo(); - Value *Ptr = - constructPointer(PointeeTy, PrivType, &Base, - PrivStructLayout->getElementOffset(u), IRB, DL); + Value *Ptr = constructPointer( + PrivType, &Base, PrivStructLayout->getElementOffset(u), IRB, DL); new StoreInst(F.getArg(ArgNo + u), Ptr, &IP); } } else if (auto *PrivArrayType = dyn_cast(PrivType)) { Type *PointeeTy = PrivArrayType->getElementType(); - Type *PointeePtrTy = PointeeTy->getPointerTo(); uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy); for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) { - Value *Ptr = constructPointer(PointeePtrTy, PrivType, &Base, - u * PointeeTySize, IRB, DL); + Value *Ptr = + constructPointer(PrivType, &Base, u * PointeeTySize, IRB, DL); new StoreInst(F.getArg(ArgNo + u), Ptr, &IP); } } else { @@ -7524,19 +7516,13 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { IRBuilder IRB(IP); const DataLayout &DL = IP->getModule()->getDataLayout(); - Type *PrivPtrType = PrivType->getPointerTo(); - if (Base->getType() != PrivPtrType) - Base = BitCastInst::CreatePointerBitCastOrAddrSpaceCast( - Base, PrivPtrType, "", ACS.getInstruction()); - // Traverse the type, build GEPs and loads. if (auto *PrivStructType = dyn_cast(PrivType)) { const StructLayout *PrivStructLayout = DL.getStructLayout(PrivStructType); for (unsigned u = 0, e = PrivStructType->getNumElements(); u < e; u++) { Type *PointeeTy = PrivStructType->getElementType(u); - Value *Ptr = - constructPointer(PointeeTy->getPointerTo(), PrivType, Base, - PrivStructLayout->getElementOffset(u), IRB, DL); + Value *Ptr = constructPointer( + PrivType, Base, PrivStructLayout->getElementOffset(u), IRB, DL); LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP); L->setAlignment(Alignment); ReplacementValues.push_back(L); @@ -7544,10 +7530,9 @@ struct AAPrivatizablePtrArgument final : public AAPrivatizablePtrImpl { } else if (auto *PrivArrayType = dyn_cast(PrivType)) { Type *PointeeTy = PrivArrayType->getElementType(); uint64_t PointeeTySize = DL.getTypeStoreSize(PointeeTy); - Type *PointeePtrTy = PointeeTy->getPointerTo(); for (unsigned u = 0, e = PrivArrayType->getNumElements(); u < e; u++) { - Value *Ptr = constructPointer(PointeePtrTy, PrivType, Base, - u * PointeeTySize, IRB, DL); + Value *Ptr = + constructPointer(PrivType, Base, u * PointeeTySize, IRB, DL); LoadInst *L = new LoadInst(PointeeTy, Ptr, "", IP); L->setAlignment(Alignment); ReplacementValues.push_back(L); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp index 39466a9e3751..6002f599ca71 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp @@ -2127,10 +2127,10 @@ Instruction *InstCombinerImpl::foldBinOpOfDisplacedShifts(BinaryOperator &I) { Constant *ShiftedC1, *ShiftedC2, *AddC; Type *Ty = I.getType(); unsigned BitWidth = Ty->getScalarSizeInBits(); - if (!match(&I, - m_c_BinOp(m_Shift(m_ImmConstant(ShiftedC1), m_Value(ShAmt)), - m_Shift(m_ImmConstant(ShiftedC2), - m_Add(m_Deferred(ShAmt), m_ImmConstant(AddC)))))) + if (!match(&I, m_c_BinOp(m_Shift(m_ImmConstant(ShiftedC1), m_Value(ShAmt)), + m_Shift(m_ImmConstant(ShiftedC2), + m_AddLike(m_Deferred(ShAmt), + m_ImmConstant(AddC)))))) return nullptr; // Make sure the add constant is a valid shift amount. diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index a991f0906052..255ce6973a16 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -514,6 +514,8 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) { return IC.replaceInstUsesWith(II, ConstantInt::getNullValue(II.getType())); } + Constant *C; + if (IsTZ) { // cttz(-x) -> cttz(x) if (match(Op0, m_Neg(m_Value(X)))) @@ -549,6 +551,38 @@ static Instruction *foldCttzCtlz(IntrinsicInst &II, InstCombinerImpl &IC) { if (match(Op0, m_Intrinsic(m_Value(X)))) return IC.replaceOperand(II, 0, X); + + // cttz(shl(%const, %val), 1) --> add(cttz(%const, 1), %val) + if (match(Op0, m_Shl(m_ImmConstant(C), m_Value(X))) && + match(Op1, m_One())) { + Value *ConstCttz = + IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, C, Op1); + return BinaryOperator::CreateAdd(ConstCttz, X); + } + + // cttz(lshr exact (%const, %val), 1) --> sub(cttz(%const, 1), %val) + if (match(Op0, m_Exact(m_LShr(m_ImmConstant(C), m_Value(X)))) && + match(Op1, m_One())) { + Value *ConstCttz = + IC.Builder.CreateBinaryIntrinsic(Intrinsic::cttz, C, Op1); + return BinaryOperator::CreateSub(ConstCttz, X); + } + } else { + // ctlz(lshr(%const, %val), 1) --> add(ctlz(%const, 1), %val) + if (match(Op0, m_LShr(m_ImmConstant(C), m_Value(X))) && + match(Op1, m_One())) { + Value *ConstCtlz = + IC.Builder.CreateBinaryIntrinsic(Intrinsic::ctlz, C, Op1); + return BinaryOperator::CreateAdd(ConstCtlz, X); + } + + // ctlz(shl nuw (%const, %val), 1) --> sub(ctlz(%const, 1), %val) + if (match(Op0, m_NUWShl(m_ImmConstant(C), m_Value(X))) && + match(Op1, m_One())) { + Value *ConstCtlz = + IC.Builder.CreateBinaryIntrinsic(Intrinsic::ctlz, C, Op1); + return BinaryOperator::CreateSub(ConstCtlz, X); + } } KnownBits Known = IC.computeKnownBits(Op0, 0, &II); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp index 4295c7692e6d..91642e3babce 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCompares.cpp @@ -23,7 +23,6 @@ #include "llvm/Analysis/VectorUtils.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/PatternMatch.h" #include "llvm/Support/KnownBits.h" @@ -812,8 +811,8 @@ Instruction *InstCombinerImpl::foldGEPICmp(GEPOperator *GEPLHS, Value *RHS, // Only lower this if the icmp is the only user of the GEP or if we expect // the result to fold to a constant! if ((GEPsInBounds || CmpInst::isEquality(Cond)) && - (isa(GEPLHS) || GEPLHS->hasOneUse()) && - (isa(GEPRHS) || GEPRHS->hasOneUse())) { + (GEPLHS->hasAllConstantIndices() || GEPLHS->hasOneUse()) && + (GEPRHS->hasAllConstantIndices() || GEPRHS->hasOneUse())) { // ((gep Ptr, OFFSET1) cmp (gep Ptr, OFFSET2) ---> (OFFSET1 cmp OFFSET2) Value *L = EmitGEPOffset(GEPLHS); Value *R = EmitGEPOffset(GEPRHS); @@ -7034,7 +7033,7 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) { return Res; { - Value *X, *Y, *Z; + Value *X, *Y; // Transform (X & ~Y) == 0 --> (X & Y) != 0 // and (X & ~Y) != 0 --> (X & Y) == 0 // if A is a power of 2. @@ -7044,32 +7043,18 @@ Instruction *InstCombinerImpl::visitICmpInst(ICmpInst &I) { return new ICmpInst(I.getInversePredicate(), Builder.CreateAnd(X, Y), Op1); - // Transform (~X ^ Y) s< ~Z --> (X ^ Y) s> Z, - // (~X ^ Y) s> ~Z --> (X ^ Y) s< Z, - // (~X ^ Y) s<= ~Z --> (X ^ Y) s>= Z, - // (~X ^ Y) s>= ~Z --> (X ^ Y) s<= Z, - // (~X ^ Y) u< ~Z --> (X ^ Y) u< Z, - // (~X ^ Y) u> ~Z --> (X ^ Y) u< Z, - // (~X ^ Y) u<= ~Z --> (X ^ Y) u>= Z, - // (~X ^ Y) u>= ~Z --> (X ^ Y) u<= Z, - // (~X ^ Y) == ~Z --> (X ^ Y) == Z, - // and (~X ^ Y) != ~Z --> (X ^ Y) != Z, - if (match(&I, m_c_ICmp(Pred, m_c_Xor(m_Not(m_Value(X)), m_Value(Y)), - m_Not(m_Value(Z)))) && - (I.getOperand(0)->hasOneUse() || I.getOperand(1)->hasOneUse())) - return new ICmpInst(I.getSwappedPredicate(Pred), Builder.CreateXor(X, Y), - Z); - - // ~X < ~Y --> Y < X - // ~X < C --> X > ~C - if (match(Op0, m_Not(m_Value(X)))) { - if (match(Op1, m_Not(m_Value(Y)))) - return new ICmpInst(I.getPredicate(), Y, X); - - const APInt *C; - if (match(Op1, m_APInt(C))) - return new ICmpInst(I.getSwappedPredicate(), X, - ConstantInt::get(Op1->getType(), ~(*C))); + // Op0 pred Op1 -> ~Op1 pred ~Op0, if this allows us to drop an instruction. + if (Op0->getType()->isIntOrIntVectorTy()) { + bool ConsumesOp0, ConsumesOp1; + if (isFreeToInvert(Op0, Op0->hasOneUse(), ConsumesOp0) && + isFreeToInvert(Op1, Op1->hasOneUse(), ConsumesOp1) && + (ConsumesOp0 || ConsumesOp1)) { + Value *InvOp0 = getFreelyInverted(Op0, Op0->hasOneUse(), &Builder); + Value *InvOp1 = getFreelyInverted(Op1, Op1->hasOneUse(), &Builder); + assert(InvOp0 && InvOp1 && + "Mismatch between isFreeToInvert and getFreelyInverted"); + return new ICmpInst(I.getSwappedPredicate(), InvOp0, InvOp1); + } } Instruction *AddI = nullptr; diff --git a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp index b72b68c68d98..bb2a77daa60a 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineLoadStoreAlloca.cpp @@ -213,29 +213,10 @@ static Instruction *simplifyAllocaArraySize(InstCombinerImpl &IC, AllocaInst *New = IC.Builder.CreateAlloca(NewTy, AI.getAddressSpace(), nullptr, AI.getName()); New->setAlignment(AI.getAlign()); + New->setUsedWithInAlloca(AI.isUsedWithInAlloca()); replaceAllDbgUsesWith(AI, *New, *New, DT); - - // Scan to the end of the allocation instructions, to skip over a block of - // allocas if possible...also skip interleaved debug info - // - BasicBlock::iterator It(New); - while (isa(*It) || isa(*It)) - ++It; - - // Now that I is pointing to the first non-allocation-inst in the block, - // insert our getelementptr instruction... - // - Type *IdxTy = IC.getDataLayout().getIndexType(AI.getType()); - Value *NullIdx = Constant::getNullValue(IdxTy); - Value *Idx[2] = {NullIdx, NullIdx}; - Instruction *GEP = GetElementPtrInst::CreateInBounds( - NewTy, New, Idx, New->getName() + ".sub"); - IC.InsertNewInstBefore(GEP, It); - - // Now make everything use the getelementptr instead of the original - // allocation. - return IC.replaceInstUsesWith(AI, GEP); + return IC.replaceInstUsesWith(AI, New); } } diff --git a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp index 588bb00462d2..8d5866e98a8e 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineMulDivRem.cpp @@ -300,8 +300,7 @@ Instruction *InstCombinerImpl::visitMul(BinaryOperator &I) { // Canonicalize (X|C1)*MulC -> X*MulC+C1*MulC. Value *X; Constant *C1; - if (match(Op0, m_OneUse(m_Add(m_Value(X), m_ImmConstant(C1)))) || - match(Op0, m_OneUse(m_DisjointOr(m_Value(X), m_ImmConstant(C1))))) { + if (match(Op0, m_OneUse(m_AddLike(m_Value(X), m_ImmConstant(C1))))) { // C1*MulC simplifies to a tidier constant. Value *NewC = Builder.CreateMul(C1, MulC); auto *BOp0 = cast(Op0); diff --git a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp index 513b185c83a4..62e49469cb01 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineNegator.cpp @@ -43,14 +43,11 @@ #include #include #include -#include #include #include namespace llvm { -class AssumptionCache; class DataLayout; -class DominatorTree; class LLVMContext; } // namespace llvm diff --git a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp index 30e058000d2b..f072f5cec309 100644 --- a/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp +++ b/llvm/lib/Transforms/InstCombine/InstructionCombining.cpp @@ -356,6 +356,7 @@ static bool simplifyAssocCastAssoc(BinaryOperator *BinOp1, IC.replaceOperand(*Cast, 0, BinOp2->getOperand(0)); IC.replaceOperand(*BinOp1, 1, FoldedC); + BinOp1->dropPoisonGeneratingFlags(); Cast->dropPoisonGeneratingFlags(); return true; } diff --git a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp index da157c966bfc..b175e6f93f3e 100644 --- a/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/AddressSanitizer.cpp @@ -2144,6 +2144,9 @@ ModuleAddressSanitizer::CreateMetadataGlobal(Module &M, Constant *Initializer, M, Initializer->getType(), false, Linkage, Initializer, Twine("__asan_global_") + GlobalValue::dropLLVMManglingEscape(OriginalName)); Metadata->setSection(getGlobalMetadataSection()); + // Place metadata in a large section for x86-64 ELF binaries to mitigate + // relocation pressure. + setGlobalVariableLargeSection(TargetTriple, *Metadata); return Metadata; } diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp index 10258e254679..adb4ffd4c812 100644 --- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp +++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp @@ -49,6 +49,7 @@ #include "llvm/Support/Error.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/TargetParser/Triple.h" +#include "llvm/Transforms/Instrumentation.h" #include "llvm/Transforms/Instrumentation/PGOInstrumentation.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/ModuleUtils.h" @@ -406,13 +407,15 @@ enum class ValueProfilingCallType { } // end anonymous namespace -PreservedAnalyses InstrProfiling::run(Module &M, ModuleAnalysisManager &AM) { +PreservedAnalyses InstrProfilingLoweringPass::run(Module &M, + ModuleAnalysisManager &AM) { FunctionAnalysisManager &FAM = AM.getResult(M).getManager(); auto GetTLI = [&FAM](Function &F) -> TargetLibraryInfo & { return FAM.getResult(F); }; - if (!run(M, GetTLI)) + InstrProfiling Lowerer(M, Options, GetTLI, IsCS); + if (!Lowerer.lower()) return PreservedAnalyses::all(); return PreservedAnalyses::none(); @@ -534,17 +537,7 @@ static bool containsProfilingIntrinsics(Module &M) { containsIntrinsic(llvm::Intrinsic::instrprof_value_profile); } -bool InstrProfiling::run( - Module &M, std::function GetTLI) { - this->M = &M; - this->GetTLI = std::move(GetTLI); - NamesVar = nullptr; - NamesSize = 0; - ProfileDataMap.clear(); - CompilerUsedVars.clear(); - UsedVars.clear(); - TT = Triple(M.getTargetTriple()); - +bool InstrProfiling::lower() { bool MadeChange = false; bool NeedsRuntimeHook = needsRuntimeHookUnconditionally(TT); if (NeedsRuntimeHook) @@ -677,12 +670,12 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) { Ind->getOperandBundlesAsDefs(OpBundles); if (!IsMemOpSize) { Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)}; - Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args, + Call = Builder.CreateCall(getOrInsertValueProfilingCall(M, *TLI), Args, OpBundles); } else { Value *Args[3] = {Ind->getTargetValue(), DataVar, Builder.getInt32(Index)}; Call = Builder.CreateCall( - getOrInsertValueProfilingCall(*M, *TLI, ValueProfilingCallType::MemOp), + getOrInsertValueProfilingCall(M, *TLI, ValueProfilingCallType::MemOp), Args, OpBundles); } if (auto AK = TLI->getExtAttrForI32Param(false)) @@ -704,18 +697,18 @@ Value *InstrProfiling::getCounterAddress(InstrProfCntrInstBase *I) { if (!isRuntimeCounterRelocationEnabled()) return Addr; - Type *Int64Ty = Type::getInt64Ty(M->getContext()); + Type *Int64Ty = Type::getInt64Ty(M.getContext()); Function *Fn = I->getParent()->getParent(); LoadInst *&BiasLI = FunctionToProfileBiasMap[Fn]; if (!BiasLI) { IRBuilder<> EntryBuilder(&Fn->getEntryBlock().front()); - auto *Bias = M->getGlobalVariable(getInstrProfCounterBiasVarName()); + auto *Bias = M.getGlobalVariable(getInstrProfCounterBiasVarName()); if (!Bias) { // Compiler must define this variable when runtime counter relocation // is being used. Runtime has a weak external reference that is used // to check whether that's the case or not. Bias = new GlobalVariable( - *M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage, + M, Int64Ty, false, GlobalValue::LinkOnceODRLinkage, Constant::getNullValue(Int64Ty), getInstrProfCounterBiasVarName()); Bias->setVisibility(GlobalVariable::HiddenVisibility); // A definition that's weak (linkonce_odr) without being in a COMDAT @@ -723,7 +716,7 @@ Value *InstrProfiling::getCounterAddress(InstrProfCntrInstBase *I) { // data word from every TU but one. Putting it in COMDAT ensures there // will be exactly one data slot in the link. if (TT.supportsCOMDAT()) - Bias->setComdat(M->getOrInsertComdat(Bias->getName())); + Bias->setComdat(M.getOrInsertComdat(Bias->getName())); } BiasLI = EntryBuilder.CreateLoad(Int64Ty, Bias); } @@ -739,9 +732,9 @@ Value *InstrProfiling::getBitmapAddress(InstrProfMCDCTVBitmapUpdate *I) { Bitmaps->getValueType(), Bitmaps, 0, I->getBitmapIndex()->getZExtValue()); if (isRuntimeCounterRelocationEnabled()) { - LLVMContext &Ctx = M->getContext(); + LLVMContext &Ctx = M.getContext(); Ctx.diagnose(DiagnosticInfoPGOProfile( - M->getName().data(), + M.getName().data(), Twine("Runtime counter relocation is presently not supported for MC/DC " "bitmaps."), DS_Warning)); @@ -762,12 +755,12 @@ void InstrProfiling::lowerTimestamp( InstrProfTimestampInst *TimestampInstruction) { assert(TimestampInstruction->getIndex()->isZeroValue() && "timestamp probes are always the first probe for a function"); - auto &Ctx = M->getContext(); + auto &Ctx = M.getContext(); auto *TimestampAddr = getCounterAddress(TimestampInstruction); IRBuilder<> Builder(TimestampInstruction); auto *CalleeTy = FunctionType::get(Type::getVoidTy(Ctx), TimestampAddr->getType(), false); - auto Callee = M->getOrInsertFunction( + auto Callee = M.getOrInsertFunction( INSTR_PROF_QUOTE(INSTR_PROF_PROFILE_SET_TIMESTAMP), CalleeTy); Builder.CreateCall(Callee, {TimestampAddr}); TimestampInstruction->eraseFromParent(); @@ -812,10 +805,10 @@ void InstrProfiling::lowerCoverageData(GlobalVariable *CoverageNamesVar) { void InstrProfiling::lowerMCDCTestVectorBitmapUpdate( InstrProfMCDCTVBitmapUpdate *Update) { IRBuilder<> Builder(Update); - auto *Int8Ty = Type::getInt8Ty(M->getContext()); - auto *Int8PtrTy = PointerType::getUnqual(M->getContext()); - auto *Int32Ty = Type::getInt32Ty(M->getContext()); - auto *Int64Ty = Type::getInt64Ty(M->getContext()); + auto *Int8Ty = Type::getInt8Ty(M.getContext()); + auto *Int8PtrTy = PointerType::getUnqual(M.getContext()); + auto *Int32Ty = Type::getInt32Ty(M.getContext()); + auto *Int64Ty = Type::getInt64Ty(M.getContext()); auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); auto *BitmapAddr = getBitmapAddress(Update); @@ -864,7 +857,7 @@ void InstrProfiling::lowerMCDCTestVectorBitmapUpdate( void InstrProfiling::lowerMCDCCondBitmapUpdate( InstrProfMCDCCondBitmapUpdate *Update) { IRBuilder<> Builder(Update); - auto *Int32Ty = Type::getInt32Ty(M->getContext()); + auto *Int32Ty = Type::getInt32Ty(M.getContext()); auto *MCDCCondBitmapAddr = Update->getMCDCCondBitmapAddr(); // Load the MCDC temporary value from the stack. @@ -1046,8 +1039,8 @@ static bool needsRuntimeRegistrationOfSectionRange(const Triple &TT) { void InstrProfiling::maybeSetComdat(GlobalVariable *GV, Function *Fn, StringRef VarName) { - bool DataReferencedByCode = profDataReferencedByCode(*M); - bool NeedComdat = needsComdatForCounter(*Fn, *M); + bool DataReferencedByCode = profDataReferencedByCode(M); + bool NeedComdat = needsComdatForCounter(*Fn, M); bool UseComdat = (NeedComdat || TT.isOSBinFormatELF()); if (!UseComdat) @@ -1055,7 +1048,7 @@ void InstrProfiling::maybeSetComdat(GlobalVariable *GV, Function *Fn, StringRef GroupName = TT.isOSBinFormatCOFF() && DataReferencedByCode ? GV->getName() : VarName; - Comdat *C = M->getOrInsertComdat(GroupName); + Comdat *C = M.getOrInsertComdat(GroupName); if (!NeedComdat) C->setSelectionKind(Comdat::NoDeduplicate); GV->setComdat(C); @@ -1140,8 +1133,8 @@ InstrProfiling::createRegionBitmaps(InstrProfMCDCBitmapInstBase *Inc, StringRef Name, GlobalValue::LinkageTypes Linkage) { uint64_t NumBytes = Inc->getNumBitmapBytes()->getZExtValue(); - auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M->getContext()), NumBytes); - auto GV = new GlobalVariable(*M, BitmapTy, false, Linkage, + auto *BitmapTy = ArrayType::get(Type::getInt8Ty(M.getContext()), NumBytes); + auto GV = new GlobalVariable(M, BitmapTy, false, Linkage, Constant::getNullValue(BitmapTy), Name); GV->setAlignment(Align(1)); return GV; @@ -1166,7 +1159,7 @@ GlobalVariable * InstrProfiling::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name, GlobalValue::LinkageTypes Linkage) { uint64_t NumCounters = Inc->getNumCounters()->getZExtValue(); - auto &Ctx = M->getContext(); + auto &Ctx = M.getContext(); GlobalVariable *GV; if (isa(Inc)) { auto *CounterTy = Type::getInt8Ty(Ctx); @@ -1174,13 +1167,13 @@ InstrProfiling::createRegionCounters(InstrProfCntrInstBase *Inc, StringRef Name, // TODO: `Constant::getAllOnesValue()` does not yet accept an array type. std::vector InitialValues(NumCounters, Constant::getAllOnesValue(CounterTy)); - GV = new GlobalVariable(*M, CounterArrTy, false, Linkage, + GV = new GlobalVariable(M, CounterArrTy, false, Linkage, ConstantArray::get(CounterArrTy, InitialValues), Name); GV->setAlignment(Align(1)); } else { auto *CounterTy = ArrayType::get(Type::getInt64Ty(Ctx), NumCounters); - GV = new GlobalVariable(*M, CounterTy, false, Linkage, + GV = new GlobalVariable(M, CounterTy, false, Linkage, Constant::getNullValue(CounterTy), Name); GV->setAlignment(Align(8)); } @@ -1200,10 +1193,10 @@ InstrProfiling::getOrCreateRegionCounters(InstrProfCntrInstBase *Inc) { PD.RegionCounters = CounterPtr; if (DebugInfoCorrelate) { - LLVMContext &Ctx = M->getContext(); + LLVMContext &Ctx = M.getContext(); Function *Fn = Inc->getParent()->getParent(); if (auto *SP = Fn->getSubprogram()) { - DIBuilder DB(*M, true, SP->getUnit()); + DIBuilder DB(M, true, SP->getUnit()); Metadata *FunctionNameAnnotation[] = { MDString::get(Ctx, InstrProfCorrelator::FunctionNameAttributeName), MDString::get(Ctx, getPGOFuncNameVarInitializer(NamePtr)), @@ -1254,7 +1247,7 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) { if (PD.DataVar) return; - LLVMContext &Ctx = M->getContext(); + LLVMContext &Ctx = M.getContext(); Function *Fn = Inc->getParent()->getParent(); GlobalValue::LinkageTypes Linkage = NamePtr->getLinkage(); @@ -1270,8 +1263,8 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) { Visibility = GlobalValue::DefaultVisibility; } - bool DataReferencedByCode = profDataReferencedByCode(*M); - bool NeedComdat = needsComdatForCounter(*Fn, *M); + bool DataReferencedByCode = profDataReferencedByCode(M); + bool NeedComdat = needsComdatForCounter(*Fn, M); bool Renamed; // The Data Variable section is anchored to profile counters. @@ -1291,9 +1284,10 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) { !needsRuntimeRegistrationOfSectionRange(TT)) { ArrayType *ValuesTy = ArrayType::get(Type::getInt64Ty(Ctx), NS); auto *ValuesVar = new GlobalVariable( - *M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy), + M, ValuesTy, false, Linkage, Constant::getNullValue(ValuesTy), getVarName(Inc, getInstrProfValuesVarPrefix(), Renamed)); ValuesVar->setVisibility(Visibility); + setGlobalVariableLargeSection(TT, *ValuesVar); ValuesVar->setSection( getInstrProfSectionName(IPSK_vals, TT.getObjectFormat())); ValuesVar->setAlignment(Align(8)); @@ -1307,7 +1301,7 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) { uint64_t NumBitmapBytes = PD.NumBitmapBytes; // Create data variable. - auto *IntPtrTy = M->getDataLayout().getIntPtrType(M->getContext()); + auto *IntPtrTy = M.getDataLayout().getIntPtrType(M.getContext()); auto *Int16Ty = Type::getInt16Ty(Ctx); auto *Int16ArrayTy = ArrayType::get(Int16Ty, IPVK_Last + 1); Type *DataTypes[] = { @@ -1340,7 +1334,7 @@ void InstrProfiling::createDataVariable(InstrProfCntrInstBase *Inc) { Visibility = GlobalValue::DefaultVisibility; } auto *Data = - new GlobalVariable(*M, DataTy, false, Linkage, nullptr, DataVarName); + new GlobalVariable(M, DataTy, false, Linkage, nullptr, DataVarName); // Reference the counter variable with a label difference (link-time // constant). auto *RelativeCounterPtr = @@ -1411,7 +1405,7 @@ void InstrProfiling::emitVNodes() { if (NumCounters < INSTR_PROF_MIN_VAL_COUNTS) NumCounters = std::max(INSTR_PROF_MIN_VAL_COUNTS, (int)NumCounters * 2); - auto &Ctx = M->getContext(); + auto &Ctx = M.getContext(); Type *VNodeTypes[] = { #define INSTR_PROF_VALUE_NODE(Type, LLVMType, Name, Init) LLVMType, #include "llvm/ProfileData/InstrProfData.inc" @@ -1420,11 +1414,12 @@ void InstrProfiling::emitVNodes() { ArrayType *VNodesTy = ArrayType::get(VNodeTy, NumCounters); auto *VNodesVar = new GlobalVariable( - *M, VNodesTy, false, GlobalValue::PrivateLinkage, + M, VNodesTy, false, GlobalValue::PrivateLinkage, Constant::getNullValue(VNodesTy), getInstrProfVNodesVarName()); + setGlobalVariableLargeSection(TT, *VNodesVar); VNodesVar->setSection( getInstrProfSectionName(IPSK_vnodes, TT.getObjectFormat())); - VNodesVar->setAlignment(M->getDataLayout().getABITypeAlign(VNodesTy)); + VNodesVar->setAlignment(M.getDataLayout().getABITypeAlign(VNodesTy)); // VNodesVar is used by runtime but not referenced via relocation by other // sections. Conservatively make it linker retained. UsedVars.push_back(VNodesVar); @@ -1442,13 +1437,14 @@ void InstrProfiling::emitNameData() { report_fatal_error(Twine(toString(std::move(E))), false); } - auto &Ctx = M->getContext(); + auto &Ctx = M.getContext(); auto *NamesVal = ConstantDataArray::getString(Ctx, StringRef(CompressedNameStr), false); - NamesVar = new GlobalVariable(*M, NamesVal->getType(), true, + NamesVar = new GlobalVariable(M, NamesVal->getType(), true, GlobalValue::PrivateLinkage, NamesVal, getInstrProfNamesVarName()); NamesSize = CompressedNameStr.size(); + setGlobalVariableLargeSection(TT, *NamesVar); NamesVar->setSection( getInstrProfSectionName(IPSK_name, TT.getObjectFormat())); // On COFF, it's important to reduce the alignment down to 1 to prevent the @@ -1468,9 +1464,9 @@ void InstrProfiling::emitRegistration() { return; // Construct the function. - auto *VoidTy = Type::getVoidTy(M->getContext()); - auto *VoidPtrTy = PointerType::getUnqual(M->getContext()); - auto *Int64Ty = Type::getInt64Ty(M->getContext()); + auto *VoidTy = Type::getVoidTy(M.getContext()); + auto *VoidPtrTy = PointerType::getUnqual(M.getContext()); + auto *Int64Ty = Type::getInt64Ty(M.getContext()); auto *RegisterFTy = FunctionType::get(VoidTy, false); auto *RegisterF = Function::Create(RegisterFTy, GlobalValue::InternalLinkage, getInstrProfRegFuncsName(), M); @@ -1483,7 +1479,7 @@ void InstrProfiling::emitRegistration() { Function::Create(RuntimeRegisterTy, GlobalVariable::ExternalLinkage, getInstrProfRegFuncName(), M); - IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", RegisterF)); + IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", RegisterF)); for (Value *Data : CompilerUsedVars) if (!isa(Data)) IRB.CreateCall(RuntimeRegisterF, Data); @@ -1511,13 +1507,13 @@ bool InstrProfiling::emitRuntimeHook() { return false; // If the module's provided its own runtime, we don't need to do anything. - if (M->getGlobalVariable(getInstrProfRuntimeHookVarName())) + if (M.getGlobalVariable(getInstrProfRuntimeHookVarName())) return false; // Declare an external variable that will pull in the runtime initialization. - auto *Int32Ty = Type::getInt32Ty(M->getContext()); + auto *Int32Ty = Type::getInt32Ty(M.getContext()); auto *Var = - new GlobalVariable(*M, Int32Ty, false, GlobalValue::ExternalLinkage, + new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage, nullptr, getInstrProfRuntimeHookVarName()); Var->setVisibility(GlobalValue::HiddenVisibility); @@ -1534,9 +1530,9 @@ bool InstrProfiling::emitRuntimeHook() { User->addFnAttr(Attribute::NoRedZone); User->setVisibility(GlobalValue::HiddenVisibility); if (TT.supportsCOMDAT()) - User->setComdat(M->getOrInsertComdat(User->getName())); + User->setComdat(M.getOrInsertComdat(User->getName())); - IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", User)); + IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", User)); auto *Load = IRB.CreateLoad(Int32Ty, Var); IRB.CreateRet(Load); @@ -1557,15 +1553,15 @@ void InstrProfiling::emitUses() { // and ensure this GC property as well. Otherwise, we have to conservatively // make all of the sections retained by the linker. if (TT.isOSBinFormatELF() || TT.isOSBinFormatMachO() || - (TT.isOSBinFormatCOFF() && !profDataReferencedByCode(*M))) - appendToCompilerUsed(*M, CompilerUsedVars); + (TT.isOSBinFormatCOFF() && !profDataReferencedByCode(M))) + appendToCompilerUsed(M, CompilerUsedVars); else - appendToUsed(*M, CompilerUsedVars); + appendToUsed(M, CompilerUsedVars); // We do not add proper references from used metadata sections to NamesVar and // VNodesVar, so we have to be conservative and place them in llvm.used // regardless of the target, - appendToUsed(*M, UsedVars); + appendToUsed(M, UsedVars); } void InstrProfiling::emitInitialization() { @@ -1574,13 +1570,13 @@ void InstrProfiling::emitInitialization() { // LTO/ThinLTO linking. Pass PGOInstrumentationGenCreateVar should // have already create the variable before LTO/ThinLTO linking. if (!IsCS) - createProfileFileNameVar(*M, Options.InstrProfileOutput); - Function *RegisterF = M->getFunction(getInstrProfRegFuncsName()); + createProfileFileNameVar(M, Options.InstrProfileOutput); + Function *RegisterF = M.getFunction(getInstrProfRegFuncsName()); if (!RegisterF) return; // Create the initialization function. - auto *VoidTy = Type::getVoidTy(M->getContext()); + auto *VoidTy = Type::getVoidTy(M.getContext()); auto *F = Function::Create(FunctionType::get(VoidTy, false), GlobalValue::InternalLinkage, getInstrProfInitFuncName(), M); @@ -1590,9 +1586,9 @@ void InstrProfiling::emitInitialization() { F->addFnAttr(Attribute::NoRedZone); // Add the basic block and the necessary calls. - IRBuilder<> IRB(BasicBlock::Create(M->getContext(), "", F)); + IRBuilder<> IRB(BasicBlock::Create(M.getContext(), "", F)); IRB.CreateCall(RegisterF, {}); IRB.CreateRetVoid(); - appendToGlobalCtors(*M, F, 0); + appendToGlobalCtors(M, F, 0); } diff --git a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp index 806afc8fcdf7..7a03ee46d6fd 100644 --- a/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp +++ b/llvm/lib/Transforms/Instrumentation/Instrumentation.cpp @@ -85,3 +85,10 @@ Comdat *llvm::getOrCreateFunctionComdat(Function &F, Triple &T) { return C; } +void llvm::setGlobalVariableLargeSection(const Triple &TargetTriple, + GlobalVariable &GV) { + if (TargetTriple.getArch() == Triple::x86_64 && + TargetTriple.getObjectFormat() == Triple::ELF) { + GV.setCodeModel(CodeModel::Large); + } +} diff --git a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp index 906687663519..fe672a4377a1 100644 --- a/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp +++ b/llvm/lib/Transforms/Instrumentation/SanitizerCoverage.cpp @@ -329,8 +329,7 @@ ModuleSanitizerCoverage::CreateSecStartEnd(Module &M, const char *Section, // Account for the fact that on windows-msvc __start_* symbols actually // point to a uint64_t before the start of the array. - auto SecStartI8Ptr = IRB.CreatePointerCast(SecStart, PtrTy); - auto GEP = IRB.CreateGEP(Int8Ty, SecStartI8Ptr, + auto GEP = IRB.CreateGEP(Int8Ty, SecStart, ConstantInt::get(IntptrTy, sizeof(uint64_t))); return std::make_pair(GEP, SecEnd); } @@ -838,8 +837,7 @@ void ModuleSanitizerCoverage::InjectTraceForSwitch( *CurModule, ArrayOfInt64Ty, false, GlobalVariable::InternalLinkage, ConstantArray::get(ArrayOfInt64Ty, Initializers), "__sancov_gen_cov_switch_values"); - IRB.CreateCall(SanCovTraceSwitchFunction, - {Cond, IRB.CreatePointerCast(GV, PtrTy)}); + IRB.CreateCall(SanCovTraceSwitchFunction, {Cond, GV}); } } } diff --git a/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp b/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp index 63b7903ef955..f3422a705dca 100644 --- a/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp +++ b/llvm/lib/Transforms/Scalar/AlignmentFromAssumptions.cpp @@ -83,11 +83,7 @@ static Align getNewAlignment(const SCEV *AASCEV, const SCEV *AlignSCEV, const SCEV *OffSCEV, Value *Ptr, ScalarEvolution *SE) { const SCEV *PtrSCEV = SE->getSCEV(Ptr); - // On a platform with 32-bit allocas, but 64-bit flat/global pointer sizes - // (*cough* AMDGPU), the effective SCEV type of AASCEV and PtrSCEV - // may disagree. Trunc/extend so they agree. - PtrSCEV = SE->getTruncateOrZeroExtend( - PtrSCEV, SE->getEffectiveSCEVType(AASCEV->getType())); + const SCEV *DiffSCEV = SE->getMinusSCEV(PtrSCEV, AASCEV); if (isa(DiffSCEV)) return Align(1); @@ -267,11 +263,17 @@ bool AlignmentFromAssumptionsPass::processAssumption(CallInst *ACall, // Now that we've updated that use of the pointer, look for other uses of // the pointer to update. Visited.insert(J); - for (User *UJ : J->users()) { - Instruction *K = cast(UJ); - if (!Visited.count(K)) - WorkList.push_back(K); - } + if (isa(J) || isa(J)) + for (auto &U : J->uses()) { + if (U->getType()->isPointerTy()) { + Instruction *K = cast(U.getUser()); + StoreInst *SI = dyn_cast(K); + if (SI && SI->getPointerOperandIndex() != U.getOperandNo()) + continue; + if (!Visited.count(K)) + WorkList.push_back(K); + } + } } return true; diff --git a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp index a6fbddca5cba..43d9883fcfd4 100644 --- a/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp +++ b/llvm/lib/Transforms/Scalar/ConstraintElimination.cpp @@ -26,7 +26,6 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/Dominators.h" #include "llvm/IR/Function.h" -#include "llvm/IR/GetElementPtrTypeIterator.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/InstrTypes.h" #include "llvm/IR/Instructions.h" @@ -36,7 +35,6 @@ #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/DebugCounter.h" -#include "llvm/Support/KnownBits.h" #include "llvm/Support/MathExtras.h" #include "llvm/Transforms/Utils/Cloning.h" #include "llvm/Transforms/Utils/ValueMapper.h" diff --git a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp index 5f82af1ca46d..9df28747570c 100644 --- a/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp +++ b/llvm/lib/Transforms/Scalar/InductiveRangeCheckElimination.cpp @@ -89,7 +89,6 @@ #include #include #include -#include #include #include diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp index 446aa497026d..f14541a1a037 100644 --- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp +++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp @@ -755,7 +755,7 @@ static unsigned getFullUnrollBoostingFactor(const EstimatedUnrollCost &Cost, static std::optional shouldPragmaUnroll(Loop *L, const PragmaInfo &PInfo, const unsigned TripMultiple, const unsigned TripCount, - const UnrollCostEstimator UCE, + unsigned MaxTripCount, const UnrollCostEstimator UCE, const TargetTransformInfo::UnrollingPreferences &UP) { // Using unroll pragma @@ -776,6 +776,10 @@ shouldPragmaUnroll(Loop *L, const PragmaInfo &PInfo, if (PInfo.PragmaFullUnroll && TripCount != 0) return TripCount; + if (PInfo.PragmaEnableUnroll && !TripCount && MaxTripCount && + MaxTripCount <= UnrollMaxUpperBound) + return MaxTripCount; + // if didn't return until here, should continue to other priorties return std::nullopt; } @@ -902,7 +906,7 @@ bool llvm::computeUnrollCount( // 1st priority is unroll count set by "unroll-count" option. // 2nd priority is unroll count set by pragma. if (auto UnrollFactor = shouldPragmaUnroll(L, PInfo, TripMultiple, TripCount, - UCE, UP)) { + MaxTripCount, UCE, UP)) { UP.Count = *UnrollFactor; if (UserUnrollCount || (PragmaCount > 0)) { diff --git a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp index 0e55249d63a8..9d058e0d2483 100644 --- a/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp +++ b/llvm/lib/Transforms/Scalar/MemCpyOptimizer.cpp @@ -880,8 +880,11 @@ bool MemCpyOptPass::performCallSlotOptzn(Instruction *cpyLoad, return false; const DataLayout &DL = cpyLoad->getModule()->getDataLayout(); - uint64_t srcSize = DL.getTypeAllocSize(srcAlloca->getAllocatedType()) * - srcArraySize->getZExtValue(); + TypeSize SrcAllocaSize = DL.getTypeAllocSize(srcAlloca->getAllocatedType()); + // We can't optimize scalable types. + if (SrcAllocaSize.isScalable()) + return false; + uint64_t srcSize = SrcAllocaSize * srcArraySize->getZExtValue(); if (cpySize < srcSize) return false; diff --git a/llvm/lib/Transforms/Scalar/Reassociate.cpp b/llvm/lib/Transforms/Scalar/Reassociate.cpp index 42e979db24d2..818c7b40d489 100644 --- a/llvm/lib/Transforms/Scalar/Reassociate.cpp +++ b/llvm/lib/Transforms/Scalar/Reassociate.cpp @@ -466,7 +466,8 @@ using RepeatedValue = std::pair; /// type and thus make the expression bigger. static bool LinearizeExprTree(Instruction *I, SmallVectorImpl &Ops, - ReassociatePass::OrderedSet &ToRedo) { + ReassociatePass::OrderedSet &ToRedo, + bool &HasNUW) { assert((isa(I) || isa(I)) && "Expected a UnaryOperator or BinaryOperator!"); LLVM_DEBUG(dbgs() << "LINEARIZE: " << *I << '\n'); @@ -515,6 +516,9 @@ static bool LinearizeExprTree(Instruction *I, std::pair P = Worklist.pop_back_val(); I = P.first; // We examine the operands of this binary operator. + if (isa(I)) + HasNUW &= I->hasNoUnsignedWrap(); + for (unsigned OpIdx = 0; OpIdx < I->getNumOperands(); ++OpIdx) { // Visit operands. Value *Op = I->getOperand(OpIdx); APInt Weight = P.second; // Number of paths to this operand. @@ -657,7 +661,8 @@ static bool LinearizeExprTree(Instruction *I, /// Now that the operands for this expression tree are /// linearized and optimized, emit them in-order. void ReassociatePass::RewriteExprTree(BinaryOperator *I, - SmallVectorImpl &Ops) { + SmallVectorImpl &Ops, + bool HasNUW) { assert(Ops.size() > 1 && "Single values should be used directly!"); // Since our optimizations should never increase the number of operations, the @@ -814,14 +819,20 @@ void ReassociatePass::RewriteExprTree(BinaryOperator *I, if (ExpressionChangedStart) { bool ClearFlags = true; do { - // Preserve FastMathFlags. + // Preserve flags. if (ClearFlags) { if (isa(I)) { FastMathFlags Flags = I->getFastMathFlags(); ExpressionChangedStart->clearSubclassOptionalData(); ExpressionChangedStart->setFastMathFlags(Flags); - } else + } else { ExpressionChangedStart->clearSubclassOptionalData(); + // Note that it doesn't hold for mul if one of the operands is zero. + // TODO: We can preserve NUW flag if we prove that all mul operands + // are non-zero. + if (HasNUW && ExpressionChangedStart->getOpcode() == Instruction::Add) + ExpressionChangedStart->setHasNoUnsignedWrap(); + } } if (ExpressionChangedStart == ExpressionChangedEnd) @@ -1175,7 +1186,8 @@ Value *ReassociatePass::RemoveFactorFromExpression(Value *V, Value *Factor) { return nullptr; SmallVector Tree; - MadeChange |= LinearizeExprTree(BO, Tree, RedoInsts); + bool HasNUW = true; + MadeChange |= LinearizeExprTree(BO, Tree, RedoInsts, HasNUW); SmallVector Factors; Factors.reserve(Tree.size()); for (unsigned i = 0, e = Tree.size(); i != e; ++i) { @@ -1217,7 +1229,7 @@ Value *ReassociatePass::RemoveFactorFromExpression(Value *V, Value *Factor) { if (!FoundFactor) { // Make sure to restore the operands to the expression tree. - RewriteExprTree(BO, Factors); + RewriteExprTree(BO, Factors, HasNUW); return nullptr; } @@ -1229,7 +1241,7 @@ Value *ReassociatePass::RemoveFactorFromExpression(Value *V, Value *Factor) { RedoInsts.insert(BO); V = Factors[0].Op; } else { - RewriteExprTree(BO, Factors); + RewriteExprTree(BO, Factors, HasNUW); V = BO; } @@ -2354,7 +2366,8 @@ void ReassociatePass::ReassociateExpression(BinaryOperator *I) { // First, walk the expression tree, linearizing the tree, collecting the // operand information. SmallVector Tree; - MadeChange |= LinearizeExprTree(I, Tree, RedoInsts); + bool HasNUW = true; + MadeChange |= LinearizeExprTree(I, Tree, RedoInsts, HasNUW); SmallVector Ops; Ops.reserve(Tree.size()); for (const RepeatedValue &E : Tree) @@ -2547,7 +2560,7 @@ void ReassociatePass::ReassociateExpression(BinaryOperator *I) { dbgs() << '\n'); // Now that we ordered and optimized the expressions, splat them back into // the expression tree, removing any unneeded nodes. - RewriteExprTree(I, Ops); + RewriteExprTree(I, Ops, HasNUW); } void diff --git a/llvm/lib/Transforms/Scalar/SCCP.cpp b/llvm/lib/Transforms/Scalar/SCCP.cpp index 69679b608f8d..8a491e74b91c 100644 --- a/llvm/lib/Transforms/Scalar/SCCP.cpp +++ b/llvm/lib/Transforms/Scalar/SCCP.cpp @@ -46,7 +46,6 @@ #include "llvm/Transforms/Scalar.h" #include "llvm/Transforms/Utils/Local.h" #include "llvm/Transforms/Utils/SCCPSolver.h" -#include #include using namespace llvm; diff --git a/llvm/lib/Transforms/Utils/LowerSwitch.cpp b/llvm/lib/Transforms/Utils/LowerSwitch.cpp index 227de425ff85..d1cdab7599c4 100644 --- a/llvm/lib/Transforms/Utils/LowerSwitch.cpp +++ b/llvm/lib/Transforms/Utils/LowerSwitch.cpp @@ -42,7 +42,6 @@ #include #include #include -#include #include using namespace llvm; diff --git a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp index 8bfe9e67d15e..cd3ac317cd23 100644 --- a/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp +++ b/llvm/lib/Transforms/Utils/ScalarEvolutionExpander.cpp @@ -1532,8 +1532,26 @@ Value *SCEVExpander::expand(const SCEV *S) { V = visit(S); V = fixupLCSSAFormFor(V); } else { - for (Instruction *I : DropPoisonGeneratingInsts) + for (Instruction *I : DropPoisonGeneratingInsts) { I->dropPoisonGeneratingFlagsAndMetadata(); + // See if we can re-infer from first principles any of the flags we just + // dropped. + if (auto *OBO = dyn_cast(I)) + if (auto Flags = SE.getStrengthenedNoWrapFlagsFromBinOp(OBO)) { + auto *BO = cast(I); + BO->setHasNoUnsignedWrap( + ScalarEvolution::maskFlags(*Flags, SCEV::FlagNUW) == SCEV::FlagNUW); + BO->setHasNoSignedWrap( + ScalarEvolution::maskFlags(*Flags, SCEV::FlagNSW) == SCEV::FlagNSW); + } + if (auto *NNI = dyn_cast(I)) { + auto *Src = NNI->getOperand(0); + if (isImpliedByDomCondition(ICmpInst::ICMP_SGE, Src, + Constant::getNullValue(Src->getType()), I, + DL).value_or(false)) + NNI->setNonNeg(true); + } + } } // Remember the expanded value for this SCEV at this location. // diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index c09cf9c2325c..89494a7f6497 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -15,7 +15,6 @@ #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" -#include "llvm/ADT/ScopeExit.h" #include "llvm/ADT/Sequence.h" #include "llvm/ADT/SetOperations.h" #include "llvm/ADT/SetVector.h" diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp index c0dbd52acbab..fa2459d1ca02 100644 --- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp @@ -108,7 +108,6 @@ #include #include #include -#include #include #include #include diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index c07e9f043b7b..f82e161fb846 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -6888,6 +6888,30 @@ void LoopVectorizationCostModel::setVectorizedCallDecision(ElementCount VF) { ParamsOk = false; break; } + case VFParamKind::OMP_Linear: { + Value *ScalarParam = CI->getArgOperand(Param.ParamPos); + // Find the stride for the scalar parameter in this loop and see if + // it matches the stride for the variant. + // TODO: do we need to figure out the cost of an extract to get the + // first lane? Or do we hope that it will be folded away? + ScalarEvolution *SE = PSE.getSE(); + const auto *SAR = + dyn_cast(SE->getSCEV(ScalarParam)); + + if (!SAR || SAR->getLoop() != TheLoop) { + ParamsOk = false; + break; + } + + const SCEVConstant *Step = + dyn_cast(SAR->getStepRecurrence(*SE)); + + if (!Step || + Step->getAPInt().getSExtValue() != Param.LinearStepOrPos) + ParamsOk = false; + + break; + } case VFParamKind::GlobalPredicate: UsesMask = true; break; @@ -8621,7 +8645,7 @@ static void addCanonicalIVRecipes(VPlan &Plan, Type *IdxTy, bool HasNUW, // Add a CanonicalIVIncrement{NUW} VPInstruction to increment the scalar // IV by VF * UF. auto *CanonicalIVIncrement = - new VPInstruction(VPInstruction::CanonicalIVIncrement, {CanonicalIVPHI}, + new VPInstruction(Instruction::Add, {CanonicalIVPHI, &Plan.getVFxUF()}, {HasNUW, false}, DL, "index.next"); CanonicalIVPHI->addOperand(CanonicalIVIncrement); diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index a1e3e76ac852..fe2aac78e5ab 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2883,6 +2883,10 @@ private: assert(!BundleMember && "Bundle and VL out of sync"); } else { MustGather.insert(VL.begin(), VL.end()); + // Build a map for gathered scalars to the nodes where they are used. + for (Value *V : VL) + if (!isConstant(V)) + ValueToGatherNodes.try_emplace(V).first->getSecond().insert(Last); } if (UserTreeIdx.UserTE) @@ -3754,7 +3758,40 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) { OrdersType CurrentOrder(NumScalars, NumScalars); SmallVector Positions; SmallBitVector UsedPositions(NumScalars); - const TreeEntry *STE = nullptr; + DenseMap UsedEntries; + DenseMap> ValueToEntryPos; + for (Value *V : TE.Scalars) { + if (!isa(V)) + continue; + const auto *LocalSTE = getTreeEntry(V); + if (!LocalSTE) + continue; + unsigned Lane = + std::distance(LocalSTE->Scalars.begin(), find(LocalSTE->Scalars, V)); + if (Lane >= NumScalars) + continue; + ++UsedEntries.try_emplace(LocalSTE, 0).first->getSecond(); + ValueToEntryPos.try_emplace(V, LocalSTE, Lane); + } + if (UsedEntries.empty()) + return std::nullopt; + const TreeEntry &BestSTE = + *std::max_element(UsedEntries.begin(), UsedEntries.end(), + [](const std::pair &P1, + const std::pair &P2) { + return P1.second < P2.second; + }) + ->first; + UsedEntries.erase(&BestSTE); + const TreeEntry *SecondBestSTE = nullptr; + if (!UsedEntries.empty()) + SecondBestSTE = + std::max_element(UsedEntries.begin(), UsedEntries.end(), + [](const std::pair &P1, + const std::pair &P2) { + return P1.second < P2.second; + }) + ->first; // Try to find all gathered scalars that are gets vectorized in other // vectorize node. Here we can have only one single tree vector node to // correctly identify order of the gathered scalars. @@ -3762,53 +3799,46 @@ BoUpSLP::findReusedOrderedScalars(const BoUpSLP::TreeEntry &TE) { Value *V = TE.Scalars[I]; if (!isa(V)) continue; - if (const auto *LocalSTE = getTreeEntry(V)) { - if (!STE) - STE = LocalSTE; - else if (STE != LocalSTE) - // Take the order only from the single vector node. - return std::nullopt; - unsigned Lane = - std::distance(STE->Scalars.begin(), find(STE->Scalars, V)); - if (Lane >= NumScalars) - return std::nullopt; - if (CurrentOrder[Lane] != NumScalars) { - if (Lane != I) - continue; - UsedPositions.reset(CurrentOrder[Lane]); - } - // The partial identity (where only some elements of the gather node are - // in the identity order) is good. - CurrentOrder[Lane] = I; - UsedPositions.set(I); + const auto [LocalSTE, Lane] = ValueToEntryPos.lookup(V); + if (!LocalSTE || (LocalSTE != &BestSTE && LocalSTE != SecondBestSTE)) + continue; + if (CurrentOrder[Lane] != NumScalars) { + if ((CurrentOrder[Lane] >= BestSTE.Scalars.size() || + BestSTE.Scalars[CurrentOrder[Lane]] == V) && + (Lane != I || LocalSTE == SecondBestSTE)) + continue; + UsedPositions.reset(CurrentOrder[Lane]); } + // The partial identity (where only some elements of the gather node are + // in the identity order) is good. + CurrentOrder[Lane] = I; + UsedPositions.set(I); } // Need to keep the order if we have a vector entry and at least 2 scalars or // the vectorized entry has just 2 scalars. - if (STE && (UsedPositions.count() > 1 || STE->Scalars.size() == 2)) { - auto &&IsIdentityOrder = [NumScalars](ArrayRef CurrentOrder) { - for (unsigned I = 0; I < NumScalars; ++I) - if (CurrentOrder[I] != I && CurrentOrder[I] != NumScalars) - return false; - return true; - }; - if (IsIdentityOrder(CurrentOrder)) - return OrdersType(); - auto *It = CurrentOrder.begin(); - for (unsigned I = 0; I < NumScalars;) { - if (UsedPositions.test(I)) { - ++I; - continue; - } - if (*It == NumScalars) { - *It = I; - ++I; - } - ++It; + if (BestSTE.Scalars.size() != 2 && UsedPositions.count() <= 1) + return std::nullopt; + auto IsIdentityOrder = [&](ArrayRef CurrentOrder) { + for (unsigned I = 0; I < NumScalars; ++I) + if (CurrentOrder[I] != I && CurrentOrder[I] != NumScalars) + return false; + return true; + }; + if (IsIdentityOrder(CurrentOrder)) + return OrdersType(); + auto *It = CurrentOrder.begin(); + for (unsigned I = 0; I < NumScalars;) { + if (UsedPositions.test(I)) { + ++I; + continue; + } + if (*It == NumScalars) { + *It = I; + ++I; } - return std::move(CurrentOrder); + ++It; } - return std::nullopt; + return std::move(CurrentOrder); } namespace { @@ -5597,6 +5627,18 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, InstructionsState S = getSameOpcode(VL, *TLI); + // Don't vectorize ephemeral values. + if (!EphValues.empty()) { + for (Value *V : VL) { + if (EphValues.count(V)) { + LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V + << ") is ephemeral.\n"); + newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx); + return; + } + } + } + // Gather if we hit the RecursionMaxDepth, unless this is a load (or z/sext of // a load), in which case peek through to include it in the tree, without // ballooning over-budget. @@ -5735,18 +5777,6 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, // We now know that this is a vector of instructions of the same type from // the same block. - // Don't vectorize ephemeral values. - if (!EphValues.empty()) { - for (Value *V : VL) { - if (EphValues.count(V)) { - LLVM_DEBUG(dbgs() << "SLP: The instruction (" << *V - << ") is ephemeral.\n"); - newTreeEntry(VL, std::nullopt /*not vectorized*/, S, UserTreeIdx); - return; - } - } - } - // Check if this is a duplicate of another entry. if (TreeEntry *E = getTreeEntry(S.OpValue)) { LLVM_DEBUG(dbgs() << "SLP: \tChecking bundle: " << *S.OpValue << ".\n"); @@ -8865,16 +8895,6 @@ static T *performExtractsShuffleAction( } InstructionCost BoUpSLP::getTreeCost(ArrayRef VectorizedVals) { - // Build a map for gathered scalars to the nodes where they are used. - ValueToGatherNodes.clear(); - for (const std::unique_ptr &EntryPtr : VectorizableTree) { - if (EntryPtr->State != TreeEntry::NeedToGather) - continue; - for (Value *V : EntryPtr->Scalars) - if (!isConstant(V)) - ValueToGatherNodes.try_emplace(V).first->getSecond().insert( - EntryPtr.get()); - } InstructionCost Cost = 0; LLVM_DEBUG(dbgs() << "SLP: Calculating cost for tree of size " << VectorizableTree.size() << ".\n"); diff --git a/llvm/lib/Transforms/Vectorize/VPlan.cpp b/llvm/lib/Transforms/Vectorize/VPlan.cpp index 0074532f9c68..263d9938d1f0 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlan.cpp @@ -741,6 +741,12 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV, for (unsigned Part = 0, UF = State.UF; Part < UF; ++Part) State.set(&VectorTripCount, VectorTripCountV, Part); + IRBuilder<> Builder(State.CFG.PrevBB->getTerminator()); + // FIXME: Model VF * UF computation completely in VPlan. + State.set(&VFxUF, + createStepForVF(Builder, TripCountV->getType(), State.VF, State.UF), + 0); + // When vectorizing the epilogue loop, the canonical induction start value // needs to be changed from zero to the value after the main vector loop. // FIXME: Improve modeling for canonical IV start values in the epilogue loop. @@ -752,7 +758,7 @@ void VPlan::prepareToExecute(Value *TripCountV, Value *VectorTripCountV, return isa(U) || isa(U) || cast(U)->getOpcode() == - VPInstruction::CanonicalIVIncrement; + Instruction::Add; }) && "the canonical IV should only be used by its increment or " "ScalarIVSteps when resetting the start value"); @@ -845,6 +851,13 @@ void VPlan::execute(VPTransformState *State) { #if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP) void VPlan::printLiveIns(raw_ostream &O) const { VPSlotTracker SlotTracker(this); + + if (VFxUF.getNumUsers() > 0) { + O << "\nLive-in "; + VFxUF.printAsOperand(O, SlotTracker); + O << " = VF * UF"; + } + if (VectorTripCount.getNumUsers() > 0) { O << "\nLive-in "; VectorTripCount.printAsOperand(O, SlotTracker); @@ -1237,6 +1250,8 @@ void VPSlotTracker::assignSlot(const VPValue *V) { } void VPSlotTracker::assignSlots(const VPlan &Plan) { + if (Plan.VFxUF.getNumUsers() > 0) + assignSlot(&Plan.VFxUF); assignSlot(&Plan.VectorTripCount); if (Plan.BackedgeTakenCount) assignSlot(Plan.BackedgeTakenCount); @@ -1260,6 +1275,11 @@ bool vputils::onlyFirstLaneUsed(VPValue *Def) { [Def](VPUser *U) { return U->onlyFirstLaneUsed(Def); }); } +bool vputils::onlyFirstPartUsed(VPValue *Def) { + return all_of(Def->users(), + [Def](VPUser *U) { return U->onlyFirstPartUsed(Def); }); +} + VPValue *vputils::getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr, ScalarEvolution &SE) { if (auto *Expanded = Plan.getSCEVExpansion(Expr)) diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h index 900245220448..94cb76889813 100644 --- a/llvm/lib/Transforms/Vectorize/VPlan.h +++ b/llvm/lib/Transforms/Vectorize/VPlan.h @@ -1058,9 +1058,7 @@ public: SLPStore, ActiveLaneMask, CalculateTripCountMinusVF, - CanonicalIVIncrement, - // The next op is similar to the above, but instead increment the - // canonical IV separately for each unrolled part. + // Increment the canonical IV separately for each unrolled part. CanonicalIVIncrementForPart, BranchOnCount, BranchOnCond @@ -1168,13 +1166,27 @@ public: return false; case VPInstruction::ActiveLaneMask: case VPInstruction::CalculateTripCountMinusVF: - case VPInstruction::CanonicalIVIncrement: case VPInstruction::CanonicalIVIncrementForPart: case VPInstruction::BranchOnCount: return true; }; llvm_unreachable("switch should return"); } + + /// Returns true if the recipe only uses the first part of operand \p Op. + bool onlyFirstPartUsed(const VPValue *Op) const override { + assert(is_contained(operands(), Op) && + "Op must be an operand of the recipe"); + if (getOperand(0) != Op) + return false; + switch (getOpcode()) { + default: + return false; + case VPInstruction::BranchOnCount: + return true; + }; + llvm_unreachable("switch should return"); + } }; /// VPWidenRecipe is a recipe for producing a copy of vector type its @@ -2126,6 +2138,13 @@ public: return true; } + /// Returns true if the recipe only uses the first part of operand \p Op. + bool onlyFirstPartUsed(const VPValue *Op) const override { + assert(is_contained(operands(), Op) && + "Op must be an operand of the recipe"); + return true; + } + /// Check if the induction described by \p Kind, /p Start and \p Step is /// canonical, i.e. has the same start, step (of 1), and type as the /// canonical IV. @@ -2545,6 +2564,9 @@ class VPlan { /// Represents the vector trip count. VPValue VectorTripCount; + /// Represents the loop-invariant VF * UF of the vector loop region. + VPValue VFxUF; + /// Holds a mapping between Values and their corresponding VPValue inside /// VPlan. Value2VPValueTy Value2VPValue; @@ -2624,6 +2646,9 @@ public: /// The vector trip count. VPValue &getVectorTripCount() { return VectorTripCount; } + /// Returns VF * UF of the vector loop region. + VPValue &getVFxUF() { return VFxUF; } + /// Mark the plan to indicate that using Value2VPValue is not safe any /// longer, because it may be stale. void disableValue2VPValue() { Value2VPValueEnabled = false; } @@ -3054,6 +3079,9 @@ namespace vputils { /// Returns true if only the first lane of \p Def is used. bool onlyFirstLaneUsed(VPValue *Def); +/// Returns true if only the first part of \p Def is used. +bool onlyFirstPartUsed(VPValue *Def); + /// Get or create a VPValue that corresponds to the expansion of \p Expr. If \p /// Expr is a SCEVConstant or SCEVUnknown, return a VPValue wrapping the live-in /// value. Otherwise return a VPExpandSCEVRecipe to expand \p Expr. If \p Plan's diff --git a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h index 473a7c28e48a..7276641551ae 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h +++ b/llvm/lib/Transforms/Vectorize/VPlanAnalysis.h @@ -16,12 +16,9 @@ namespace llvm { class LLVMContext; class VPValue; class VPBlendRecipe; -class VPInterleaveRecipe; class VPInstruction; -class VPReductionPHIRecipe; class VPWidenRecipe; class VPWidenCallRecipe; -class VPWidenCastRecipe; class VPWidenIntOrFpInductionRecipe; class VPWidenMemoryInstructionRecipe; struct VPWidenSelectRecipe; diff --git a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp index 1891b211a356..ffdeea80fa07 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp @@ -118,9 +118,9 @@ bool VPRecipeBase::mayHaveSideEffects() const { case VPInstructionSC: switch (cast(this)->getOpcode()) { case Instruction::ICmp: + case Instruction::Select: case VPInstruction::Not: case VPInstruction::CalculateTripCountMinusVF: - case VPInstruction::CanonicalIVIncrement: case VPInstruction::CanonicalIVIncrementForPart: return false; default: @@ -272,9 +272,16 @@ Value *VPInstruction::generateInstruction(VPTransformState &State, Builder.SetCurrentDebugLocation(getDebugLoc()); if (Instruction::isBinaryOp(getOpcode())) { + if (Part != 0 && vputils::onlyFirstPartUsed(this)) + return State.get(this, 0); + Value *A = State.get(getOperand(0), Part); Value *B = State.get(getOperand(1), Part); - return Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name); + auto *Res = + Builder.CreateBinOp((Instruction::BinaryOps)getOpcode(), A, B, Name); + if (auto *I = dyn_cast(Res)) + setFlags(I); + return Res; } switch (getOpcode()) { @@ -335,19 +342,6 @@ Value *VPInstruction::generateInstruction(VPTransformState &State, Value *Zero = ConstantInt::get(ScalarTC->getType(), 0); return Builder.CreateSelect(Cmp, Sub, Zero); } - case VPInstruction::CanonicalIVIncrement: { - if (Part == 0) { - auto *Phi = State.get(getOperand(0), 0); - // The loop step is equal to the vectorization factor (num of SIMD - // elements) times the unroll factor (num of SIMD instructions). - Value *Step = - createStepForVF(Builder, Phi->getType(), State.VF, State.UF); - return Builder.CreateAdd(Phi, Step, Name, hasNoUnsignedWrap(), - hasNoSignedWrap()); - } - return State.get(this, 0); - } - case VPInstruction::CanonicalIVIncrementForPart: { auto *IV = State.get(getOperand(0), VPIteration(0, 0)); if (Part == 0) @@ -468,9 +462,6 @@ void VPInstruction::print(raw_ostream &O, const Twine &Indent, case VPInstruction::FirstOrderRecurrenceSplice: O << "first-order splice"; break; - case VPInstruction::CanonicalIVIncrement: - O << "VF * UF +"; - break; case VPInstruction::BranchOnCond: O << "branch-on-cond"; break; @@ -802,6 +793,7 @@ void VPWidenCastRecipe::print(raw_ostream &O, const Twine &Indent, O << Indent << "WIDEN-CAST "; printAsOperand(O, SlotTracker); O << " = " << Instruction::getOpcodeName(Opcode) << " "; + printFlags(O); printOperands(O, SlotTracker); O << " to " << *getResultType(); } diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h index e8a6da8c3205..3bf91115debb 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.h +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.h @@ -22,11 +22,9 @@ class InductionDescriptor; class Instruction; class PHINode; class ScalarEvolution; -class Loop; class PredicatedScalarEvolution; class TargetLibraryInfo; class VPBuilder; -class VPRecipeBuilder; struct VPlanTransforms { /// Replaces the VPInstructions in \p Plan with corresponding diff --git a/llvm/lib/Transforms/Vectorize/VPlanValue.h b/llvm/lib/Transforms/Vectorize/VPlanValue.h index ac2883b30dc8..116acad8e8f3 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanValue.h +++ b/llvm/lib/Transforms/Vectorize/VPlanValue.h @@ -121,18 +121,11 @@ public: /// Remove a single \p User from the list of users. void removeUser(VPUser &User) { - bool Found = false; // The same user can be added multiple times, e.g. because the same VPValue // is used twice by the same VPUser. Remove a single one. - erase_if(Users, [&User, &Found](VPUser *Other) { - if (Found) - return false; - if (Other == &User) { - Found = true; - return true; - } - return false; - }); + auto *I = find(Users, &User); + if (I != Users.end()) + Users.erase(I); } typedef SmallVectorImpl::iterator user_iterator; @@ -303,6 +296,14 @@ public: "Op must be an operand of the recipe"); return false; } + + /// Returns true if the VPUser only uses the first part of operand \p Op. + /// Conservatively returns false. + virtual bool onlyFirstPartUsed(const VPValue *Op) const { + assert(is_contained(operands(), Op) && + "Op must be an operand of the recipe"); + return false; + } }; /// This class augments a recipe with a set of VPValues defined by the recipe. diff --git a/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types_opaque_ptr.ll b/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types_opaque_ptr.ll deleted file mode 100644 index e424683bbef3..000000000000 --- a/llvm/test/Analysis/LoopAccessAnalysis/depend_diff_types_opaque_ptr.ll +++ /dev/null @@ -1,179 +0,0 @@ -; RUN: opt -S -disable-output -passes='print' < %s 2>&1 | FileCheck %s - -; In the function below some of the accesses are done as float types and some -; are done as i32 types. When doing dependence analysis the type should not -; matter if it can be determined that they are the same size. - -%int_pair = type { i32, i32 } - -; CHECK-LABEL: function 'backdep_type_size_equivalence': -; CHECK-NEXT: loop: -; CHECK-NEXT: Memory dependences are safe with a maximum safe vector width of 3200 bits -; CHECK-NEXT: Dependences: -; CHECK-NEXT: Forward: -; CHECK-NEXT: %ld.f32 = load float, ptr %gep.iv, align 8 -> -; CHECK-NEXT: store i32 %indvars.iv.i32, ptr %gep.iv, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: Forward: -; CHECK-NEXT: %ld.f32 = load float, ptr %gep.iv, align 8 -> -; CHECK-NEXT: store float %val, ptr %gep.iv.min.100, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: BackwardVectorizable: -; CHECK-NEXT: store float %val, ptr %gep.iv.min.100, align 8 -> -; CHECK-NEXT: store i32 %indvars.iv.i32, ptr %gep.iv, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: Run-time memory checks: -; CHECK-NEXT: Grouped accesses: - -define void @backdep_type_size_equivalence(ptr nocapture %vec, i64 %n) { -entry: - br label %loop - -loop: - %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %loop ] - - ;; Load from vec[indvars.iv].x as float - %gep.iv = getelementptr inbounds %int_pair, ptr %vec, i64 %indvars.iv, i32 0 - %ld.f32 = load float, ptr %gep.iv, align 8 - %val = fmul fast float %ld.f32, 5.0 - - ;; Store to vec[indvars.iv - 100].x as float - %indvars.iv.min.100 = add nsw i64 %indvars.iv, -100 - %gep.iv.min.100 = getelementptr inbounds %int_pair, ptr %vec, i64 %indvars.iv.min.100, i32 0 - store float %val, ptr %gep.iv.min.100, align 8 - - ;; Store to vec[indvars.iv].x as i32, creating a backward dependency between - ;; the two stores with different element types but the same element size. - %indvars.iv.i32 = trunc i64 %indvars.iv to i32 - store i32 %indvars.iv.i32, ptr %gep.iv, align 8 - - ;; Store to vec[indvars.iv].y as i32, strided accesses should be independent - ;; between the two stores with different element types but the same element size. - %gep.iv.1 = getelementptr inbounds %int_pair, ptr %vec, i64 %indvars.iv, i32 1 - store i32 %indvars.iv.i32, ptr %gep.iv.1, align 8 - - ;; Store to vec[indvars.iv + n].y as i32, to verify no dependence in the case - ;; of unknown dependence distance. - %indvars.iv.n = add nuw nsw i64 %indvars.iv, %n - %gep.iv.n = getelementptr inbounds %int_pair, ptr %vec, i64 %indvars.iv.n, i32 1 - store i32 %indvars.iv.i32, ptr %gep.iv.n, align 8 - - ;; Loop condition. - %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 - %cond = icmp eq i64 %indvars.iv.next, %n - br i1 %cond, label %exit, label %loop - -exit: - ret void -} - -; In the function below one of the accesses is done as i19 type, which has a -; different store size than the i32 type, even though their alloc sizes are -; equivalent. This is a negative test to ensure that they are not analyzed as -; in the tests above. -; -; CHECK-LABEL: function 'backdep_type_store_size_equivalence': -; CHECK-NEXT: loop: -; CHECK-NEXT: Report: unsafe dependent memory operations in loop. -; CHECK-NEXT: Unknown data dependence. -; CHECK-NEXT: Dependences: -; CHECK-NEXT: Unknown: -; CHECK-NEXT: %ld.f32 = load float, ptr %gep.iv, align 8 -> -; CHECK-NEXT: store i19 %indvars.iv.i19, ptr %gep.iv, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: Run-time memory checks: -; CHECK-NEXT: Grouped accesses: - -define void @backdep_type_store_size_equivalence(ptr nocapture %vec, i64 %n) { -entry: - br label %loop - -loop: - %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %loop ] - - ;; Load from vec[indvars.iv].x as float - %gep.iv = getelementptr inbounds %int_pair, ptr %vec, i64 %indvars.iv, i32 0 - %ld.f32 = load float, ptr %gep.iv, align 8 - %val = fmul fast float %ld.f32, 5.0 - - ;; Store to vec[indvars.iv].x as i19. - %indvars.iv.i19 = trunc i64 %indvars.iv to i19 - store i19 %indvars.iv.i19, ptr %gep.iv, align 8 - - ;; Loop condition. - %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 - %cond = icmp eq i64 %indvars.iv.next, %n - br i1 %cond, label %exit, label %loop - -exit: - ret void -} - -; In the function below some of the accesses are done as double types and some -; are done as i64 and i32 types. This is a negative test to ensure that they -; are not analyzed as in the tests above. - -; CHECK-LABEL: function 'neg_dist_dep_type_size_equivalence': -; CHECK-NEXT: loop: -; CHECK-NEXT: Report: unsafe dependent memory operations in loop. -; CHECK-NEXT: Unknown data dependence. -; CHECK-NEXT: Dependences: -; CHECK-NEXT: Unknown: -; CHECK-NEXT: %ld.f64 = load double, ptr %gep.iv, align 8 -> -; CHECK-NEXT: store i32 %ld.i64.i32, ptr %gep.iv.n, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: Unknown: -; CHECK-NEXT: %ld.i64 = load i64, ptr %gep.iv, align 8 -> -; CHECK-NEXT: store i32 %ld.i64.i32, ptr %gep.iv.n, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: BackwardVectorizableButPreventsForwarding: -; CHECK-NEXT: %ld.f64 = load double, ptr %gep.iv, align 8 -> -; CHECK-NEXT: store double %val, ptr %gep.iv.101, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: ForwardButPreventsForwarding: -; CHECK-NEXT: store double %val, ptr %gep.iv.101, align 8 -> -; CHECK-NEXT: %ld.i64 = load i64, ptr %gep.iv, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: Unknown: -; CHECK-NEXT: store double %val, ptr %gep.iv.101, align 8 -> -; CHECK-NEXT: store i32 %ld.i64.i32, ptr %gep.iv.n, align 8 -; CHECK-EMPTY: -; CHECK-NEXT: Run-time memory checks: -; CHECK-NEXT: Grouped accesses: - -define void @neg_dist_dep_type_size_equivalence(ptr nocapture %vec, i64 %n) { -entry: - br label %loop - -loop: - %indvars.iv = phi i64 [ 0, %entry ], [ %indvars.iv.next, %loop ] - - ;; Load from vec[indvars.iv] as double - %gep.iv = getelementptr i64, ptr %vec, i64 %indvars.iv - %ld.f64 = load double, ptr %gep.iv, align 8 - %val = fmul fast double %ld.f64, 5.0 - - ;; Store to vec[indvars.iv + 101] as double - %indvars.iv.101 = add nsw i64 %indvars.iv, 101 - %gep.iv.101 = getelementptr i64, ptr %vec, i64 %indvars.iv.101 - store double %val, ptr %gep.iv.101, align 8 - - ;; Read from vec[indvars.iv] as i64 creating - ;; a forward but prevents forwarding dependence - ;; with different types but same sizes. - %ld.i64 = load i64, ptr %gep.iv, align 8 - - ;; Different sizes - %indvars.iv.n = add nuw nsw i64 %indvars.iv, %n - %gep.iv.n = getelementptr inbounds i64, ptr %vec, i64 %indvars.iv.n - %ld.i64.i32 = trunc i64 %ld.i64 to i32 - store i32 %ld.i64.i32, ptr %gep.iv.n, align 8 - - ;; Loop condition. - %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1 - %cond = icmp eq i64 %indvars.iv.next, %n - br i1 %cond, label %exit, label %loop - -exit: - ret void -} diff --git a/llvm/test/Analysis/LoopAccessAnalysis/num-iters-for-store-load-conflict.ll b/llvm/test/Analysis/LoopAccessAnalysis/num-iters-for-store-load-conflict.ll new file mode 100644 index 000000000000..d3eda21dee27 --- /dev/null +++ b/llvm/test/Analysis/LoopAccessAnalysis/num-iters-for-store-load-conflict.ll @@ -0,0 +1,310 @@ +; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes='print' -disable-output < %s 2>&1 | FileCheck %s + +define void @forward_dist_7(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_7' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 7 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @forward_dist_9(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_9' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 9 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @forward_dist_11(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_11' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 9 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @forward_dist_13(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_13' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 13 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @forward_dist_15(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_15' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 13 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @forward_dist_17(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_17' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 17 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +define void @forward_dist_19(ptr %A, ptr noalias %B) { +; CHECK-LABEL: 'forward_dist_19' +; CHECK-NEXT: loop: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Forward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: ForwardButPreventsForwarding: +; CHECK-NEXT: store i32 0, ptr %gep.2, align 4 -> +; CHECK-NEXT: %l = load i32, ptr %gep.1, align 4 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %loop + +loop: + %iv = phi i64 [ 0, %entry ], [ %next, %loop ] + %gep.1 = getelementptr i32, ptr %A, i64 %iv + %gep.2 = getelementptr i32, ptr %gep.1, i64 19 + store i32 0, ptr %gep.2, align 4 + %l = load i32, ptr %gep.1 + store i32 %l, ptr %B + %next = add nuw nsw i64 %iv, 1 + %ec = icmp eq i64 %iv, 1000 + br i1 %ec, label %exit, label %loop + +exit: + ret void +} + +@A = global [37 x [37 x double]] zeroinitializer, align 8 + +define void @unknown_loop_bounds(i64 %x, i64 %y) { +; CHECK-LABEL: 'unknown_loop_bounds' +; CHECK-NEXT: inner: +; CHECK-NEXT: Report: unsafe dependent memory operations in loop. Use #pragma clang loop distribute(enable) to allow loop distribution to attempt to isolate the offending operations into a separate loop +; CHECK-NEXT: Backward loop carried data dependence that prevents store-to-load forwarding. +; CHECK-NEXT: Dependences: +; CHECK-NEXT: BackwardVectorizableButPreventsForwarding: +; CHECK-NEXT: %l = load double, ptr %gep.0, align 8 -> +; CHECK-NEXT: store double %l, ptr %gep.1, align 8 +; CHECK-EMPTY: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; CHECK-NEXT: outer.header: +; CHECK-NEXT: Report: loop is not the innermost loop +; CHECK-NEXT: Dependences: +; CHECK-NEXT: Run-time memory checks: +; CHECK-NEXT: Grouped accesses: +; CHECK-EMPTY: +; CHECK-NEXT: Non vectorizable stores to invariant address were not found in loop. +; CHECK-NEXT: SCEV assumptions: +; CHECK-EMPTY: +; CHECK-NEXT: Expressions re-written: +; +entry: + br label %outer.header + +outer.header: + %outer.iv = phi i64 [ 0, %entry ], [ %outer.iv.next, %outer.latch ] + %outer.iv.next = add nuw nsw i64 %outer.iv, 1 + br label %inner + +inner: + %inner.iv = phi i64 [ 0, %outer.header ], [ %inner.iv.next, %inner ] + %gep.0 = getelementptr inbounds [37 x [37 x double]], ptr @A, i64 0, i64 %outer.iv, i64 %inner.iv + %l = load double, ptr %gep.0, align 8 + %gep.1 = getelementptr inbounds [37 x [37 x double]], ptr @A, i64 0, i64 %outer.iv.next, i64 %inner.iv + store double %l, ptr %gep.1, align 8 + %inner.iv.next = add nuw nsw i64 %inner.iv, 1 + %inner.ec = icmp eq i64 %inner.iv.next, %y + br i1 %inner.ec, label %outer.latch, label %inner + +outer.latch: + %outer.ec = icmp eq i64 %outer.iv.next, %x + br i1 %outer.ec, label %exit, label %outer.header + +exit: + ret void +} diff --git a/llvm/test/Analysis/ScalarEvolution/trip-count.ll b/llvm/test/Analysis/ScalarEvolution/trip-count.ll index 5973d52d05ea..22e49ebdbf4d 100644 --- a/llvm/test/Analysis/ScalarEvolution/trip-count.ll +++ b/llvm/test/Analysis/ScalarEvolution/trip-count.ll @@ -123,3 +123,28 @@ loop: leave: ret void } + +define void @non_zero_from_loop_guard(i16 %n) { +; CHECK-LABEL: 'non_zero_from_loop_guard' +; CHECK-NEXT: Determining loop execution counts for: @non_zero_from_loop_guard +; CHECK-NEXT: Loop %loop: backedge-taken count is (-1 + (1 umax (%n /u 2))) +; CHECK-NEXT: Loop %loop: constant max backedge-taken count is 32766 +; CHECK-NEXT: Loop %loop: symbolic max backedge-taken count is (-1 + (1 umax (%n /u 2))) +; CHECK-NEXT: Loop %loop: Predicated backedge-taken count is (-1 + (1 umax (%n /u 2))) +; CHECK-NEXT: Predicates: +; CHECK-NEXT: Loop %loop: Trip multiple is 1 +; +entry: + %shr = lshr i16 %n, 1 + %precond = icmp ult i16 %n, 2 + br i1 %precond, label %exit, label %loop + +loop: + %iv = phi i16 [ %inc, %loop ], [ 0, %entry ] + %inc = add nuw nsw i16 %iv, 1 + %cmp = icmp ult i16 %inc, %shr + br i1 %cmp, label %loop, label %exit + +exit: + ret void +} diff --git a/llvm/test/Analysis/ValueTracking/knownbits-and-or-xor-lowbit.ll b/llvm/test/Analysis/ValueTracking/knownbits-and-or-xor-lowbit.ll index f44948e6a089..4ca7ed9eda7b 100644 --- a/llvm/test/Analysis/ValueTracking/knownbits-and-or-xor-lowbit.ll +++ b/llvm/test/Analysis/ValueTracking/knownbits-and-or-xor-lowbit.ll @@ -147,10 +147,9 @@ define i1 @sub_YX_and_bit0_is_zero_fail(i8 %x, i8 %C) nounwind { define <2 x i1> @sub_YX_xor_bit0_is_one_fail(<2 x i8> %x, <2 x i8> %C) nounwind { ; CHECK-LABEL: @sub_YX_xor_bit0_is_one_fail( -; CHECK-NEXT: [[TMP1:%.*]] = xor <2 x i8> [[X:%.*]], -; CHECK-NEXT: [[Y:%.*]] = add <2 x i8> [[TMP1]], [[C:%.*]] -; CHECK-NEXT: [[W:%.*]] = xor <2 x i8> [[Y]], [[X]] -; CHECK-NEXT: [[R:%.*]] = icmp eq <2 x i8> [[W]], +; CHECK-NEXT: [[TMP1:%.*]] = sub <2 x i8> [[X:%.*]], [[C:%.*]] +; CHECK-NEXT: [[TMP2:%.*]] = xor <2 x i8> [[TMP1]], [[X]] +; CHECK-NEXT: [[R:%.*]] = icmp eq <2 x i8> [[TMP2]], ; CHECK-NEXT: ret <2 x i1> [[R]] ; %C1 = sub <2 x i8> %C, diff --git a/llvm/test/Assembler/constant-splat-diagnostics.ll b/llvm/test/Assembler/constant-splat-diagnostics.ll new file mode 100644 index 000000000000..464250d03ea2 --- /dev/null +++ b/llvm/test/Assembler/constant-splat-diagnostics.ll @@ -0,0 +1,40 @@ +; RUN: rm -rf %t && split-file %s %t + +; RUN: not llvm-as < %t/not_a_constant.ll -o /dev/null 2>&1 | FileCheck -check-prefix=NOT_A_CONSTANT %s +; RUN: not llvm-as < %t/not_a_sclar.ll -o /dev/null 2>&1 | FileCheck -check-prefix=NOT_A_SCALAR %s +; RUN: not llvm-as < %t/not_a_vector.ll -o /dev/null 2>&1 | FileCheck -check-prefix=NOT_A_VECTOR %s +; RUN: not llvm-as < %t/wrong_explicit_type.ll -o /dev/null 2>&1 | FileCheck -check-prefix=WRONG_EXPLICIT_TYPE %s +; RUN: not llvm-as < %t/wrong_implicit_type.ll -o /dev/null 2>&1 | FileCheck -check-prefix=WRONG_IMPLICIT_TYPE %s + +;--- not_a_constant.ll +; NOT_A_CONSTANT: error: expected instruction opcode +define <4 x i32> @not_a_constant(i32 %a) { + %splat = splat (i32 %a) + ret %splat +} + +;--- not_a_sclar.ll +; NOT_A_SCALAR: error: constant expression type mismatch: got type '<1 x i32>' but expected 'i32' +define <4 x i32> @not_a_scalar() { + ret <4 x i32> splat (<1 x i32> ) +} + +;--- not_a_vector.ll +; NOT_A_VECTOR: error: vector constant must have vector type +define <4 x i32> @not_a_vector() { + ret i32 splat (i32 7) +} + +;--- wrong_explicit_type.ll +; WRONG_EXPLICIT_TYPE: error: constant expression type mismatch: got type 'i8' but expected 'i32' +define <4 x i32> @wrong_explicit_type() { + ret <4 x i32> splat (i8 7) +} + +;--- wrong_implicit_type.ll +; WRONG_IMPLICIT_TYPE: error: constant expression type mismatch: got type 'i8' but expected 'i32' +define void @wrong_implicit_type(<4 x i32> %a) { + %add = add <4 x i32> %a, splat (i8 7) + ret void +} + diff --git a/llvm/test/Assembler/constant-splat.ll b/llvm/test/Assembler/constant-splat.ll new file mode 100644 index 000000000000..f3ec0c8340aa --- /dev/null +++ b/llvm/test/Assembler/constant-splat.ll @@ -0,0 +1,67 @@ +; RUN: llvm-as < %s | llvm-dis | llvm-as | llvm-dis | FileCheck %s + +; NOTE: Tests the expansion of the "splat" shorthand method to create vector +; constants. Future work will change how "splat" is expanded, ultimately +; leading to a point where "splat" is emitted as the disassembly. + +@my_global = external global i32 + +; CHECK: @constant.splat.i1 = constant <1 x i1> +@constant.splat.i1 = constant <1 x i1> splat (i1 true) + +; CHECK: @constant.splat.i32 = constant <5 x i32> +@constant.splat.i32 = constant <5 x i32> splat (i32 7) + +; CHECK: @constant.splat.i128 = constant <2 x i128> +@constant.splat.i128 = constant <2 x i128> splat (i128 85070591730234615870450834276742070272) + +; CHECK: @constant.splat.f16 = constant <4 x half> +@constant.splat.f16 = constant <4 x half> splat (half 0xHBC00) + +; CHECK: @constant.splat.f32 = constant <5 x float> +@constant.splat.f32 = constant <5 x float> splat (float -2.000000e+00) + +; CHECK: @constant.splat.f64 = constant <3 x double> +@constant.splat.f64 = constant <3 x double> splat (double -3.000000e+00) + +; CHECK: @constant.splat.128 = constant <2 x fp128> +@constant.splat.128 = constant <2 x fp128> splat (fp128 0xL00000000000000018000000000000000) + +; CHECK: @constant.splat.bf16 = constant <4 x bfloat> +@constant.splat.bf16 = constant <4 x bfloat> splat (bfloat 0xRC0A0) + +; CHECK: @constant.splat.x86_fp80 = constant <3 x x86_fp80> +@constant.splat.x86_fp80 = constant <3 x x86_fp80> splat (x86_fp80 0xK4000C8F5C28F5C28F800) + +; CHECK: @constant.splat.ppc_fp128 = constant <1 x ppc_fp128> +@constant.splat.ppc_fp128 = constant <1 x ppc_fp128> splat (ppc_fp128 0xM80000000000000000000000000000000) + +; CHECK: @constant.splat.global.ptr = constant <4 x ptr> +@constant.splat.global.ptr = constant <4 x ptr> splat (ptr @my_global) + +define void @add_fixed_lenth_vector_splat_i32(<4 x i32> %a) { +; CHECK: %add = add <4 x i32> %a, + %add = add <4 x i32> %a, splat (i32 137) + ret void +} + +define <4 x i32> @ret_fixed_lenth_vector_splat_i32() { +; CHECK: ret <4 x i32> + ret <4 x i32> splat (i32 56) +} + +define void @add_fixed_lenth_vector_splat_double( %a) { +; CHECK: %add = fadd %a, shufflevector ( insertelement ( poison, double 5.700000e+00, i64 0), poison, zeroinitializer) + %add = fadd %a, splat (double 5.700000e+00) + ret void +} + +define @ret_scalable_vector_splat_i32() { +; CHECK: ret shufflevector ( insertelement ( poison, i32 78, i64 0), poison, zeroinitializer) + ret splat (i32 78) +} + +define @ret_scalable_vector_ptr() { +; CHECK: ret shufflevector ( insertelement ( poison, ptr @my_global, i64 0), poison, zeroinitializer) + ret splat (ptr @my_global) +} diff --git a/llvm/test/Assembler/named-metadata.ll b/llvm/test/Assembler/named-metadata.ll index 9fa37a7989d4..32ebf2bfd26f 100644 --- a/llvm/test/Assembler/named-metadata.ll +++ b/llvm/test/Assembler/named-metadata.ll @@ -23,3 +23,7 @@ ; when emitting it, followed by xfoo. !\xfoo = !{!0, !1, !2} ; CHECK: !\5Cxfoo = !{!0, !1, !2} + +; Make sure we handle escapes with the high bit set correctly. +!\FFfoo = !{!0, !1, !2} +; CHECK: !\FFfoo = !{!0, !1, !2} diff --git a/llvm/test/Assembler/struct-ret-without-upgrade.ll b/llvm/test/Assembler/struct-ret-without-upgrade.ll index 992b2f9f767f..14f931c23abb 100644 --- a/llvm/test/Assembler/struct-ret-without-upgrade.ll +++ b/llvm/test/Assembler/struct-ret-without-upgrade.ll @@ -15,4 +15,15 @@ define %ty @test(%ty %arg) { ret %ty %copy } +define %ty @test_not_real_intrinsic() { +; CHECK-LABEL: @test_not_real_intrinsic( +; CHECK-NEXT: [[RET:%.*]] = call [[TY:%.*]] @llvm.dummy() +; CHECK-NEXT: ret [[TY]] [[RET]] +; + %ret = call %ty @llvm.dummy() + ret %ty %ret +} + +declare %ty @llvm.dummy() + declare %ty @llvm.ssa.copy.s_tys(%ty) diff --git a/llvm/test/Assembler/summary-parsing-error.ll b/llvm/test/Assembler/summary-parsing-error.ll new file mode 100644 index 000000000000..47e5c64e1298 --- /dev/null +++ b/llvm/test/Assembler/summary-parsing-error.ll @@ -0,0 +1,4 @@ +; RUN: not llvm-as < %s 2>&1 | FileCheck %s + +; CHECK: Reference to undefined global "does_not_exist" +^0 = gv: (name: "does_not_exist") diff --git a/llvm/test/Bindings/llvm-c/echo.ll b/llvm/test/Bindings/llvm-c/echo.ll index 0775cbb673e4..2e195beebd7b 100644 --- a/llvm/test/Bindings/llvm-c/echo.ll +++ b/llvm/test/Bindings/llvm-c/echo.ll @@ -269,6 +269,17 @@ exit: ret void } +define void @operandbundles() personality ptr @personalityFn { + call void @decl() [ "foo"(), "bar\00x"(i32 0, ptr null, token none) ] + invoke void @decl() [ "baz"(label %bar) ] to label %foo unwind label %bar +foo: + ret void +bar: + %1 = landingpad { ptr, i32 } + cleanup + ret void +} + define void @with_debuginfo() !dbg !4 { ret void, !dbg !7 } diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll index 575cd6b874e3..92ddc6309546 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll +++ b/llvm/test/CodeGen/AArch64/GlobalISel/arm64-irtranslator.ll @@ -1,5 +1,5 @@ -; RUN: llc -O0 -aarch64-enable-atomic-cfg-tidy=0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s -; RUN: llc -O3 -aarch64-enable-atomic-cfg-tidy=0 -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefix=O3 +; RUN: llc -O0 -aarch64-enable-atomic-cfg-tidy=0 -mattr=+lse -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s +; RUN: llc -O3 -aarch64-enable-atomic-cfg-tidy=0 -mattr=+lse -stop-after=irtranslator -global-isel -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefix=O3 ; This file checks that the translation from llvm IR to generic MachineInstr ; is correct. @@ -2077,190 +2077,147 @@ done: } ; Try a monotonic atomicrmw xchg -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_xchg(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_xchg ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_XCHG [[ADDR]](p0), [[VAL]] :: (load store monotonic (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw xchg ptr %addr, i256 1 monotonic - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_XCHG [[ADDR]](p0), [[VAL]] :: (load store monotonic (s32) on %ir.addr) + %oldval = atomicrmw xchg ptr %addr, i32 1 monotonic + ret i32 %oldval } ; Try an acquire atomicrmw add -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_add(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_add ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_ADD [[ADDR]](p0), [[VAL]] :: (load store acquire (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw add ptr %addr, i256 1 acquire - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_ADD [[ADDR]](p0), [[VAL]] :: (load store acquire (s32) on %ir.addr) + %oldval = atomicrmw add ptr %addr, i32 1 acquire + ret i32 %oldval } ; Try a release atomicrmw sub -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_sub(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_sub ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_SUB [[ADDR]](p0), [[VAL]] :: (load store release (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw sub ptr %addr, i256 1 release - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_SUB [[ADDR]](p0), [[VAL]] :: (load store release (s32) on %ir.addr) + %oldval = atomicrmw sub ptr %addr, i32 1 release + ret i32 %oldval } ; Try an acq_rel atomicrmw and -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_and(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_and ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_AND [[ADDR]](p0), [[VAL]] :: (load store acq_rel (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw and ptr %addr, i256 1 acq_rel - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc -} - -; Try an seq_cst atomicrmw nand -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_AND [[ADDR]](p0), [[VAL]] :: (load store acq_rel (s32) on %ir.addr) + %oldval = atomicrmw and ptr %addr, i32 1 acq_rel + ret i32 %oldval +} + +; Try an seq_cst atomicrmw nand. NAND isn't supported by LSE, so it +; expands to G_ATOMIC_CMPXCHG_WITH_SUCCESS. define i32 @test_atomicrmw_nand(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_nand ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): +; CHECK-NEXT: successors: %bb.2(0x80000000) ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_NAND [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw nand ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[NEG1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -1 +; CHECK-NEXT: [[OLDVALSTART:%[0-9]+]]:_(s32) = G_LOAD [[ADDR]](p0) :: (load (s32) from %ir.addr) +; CHECK: bb.2.atomicrmw.start: +; CHECK-NEXT: successors: %bb.3({{[^)]+}}), %bb.2({{[^)]+}}) +; CHECK: [[OLDVAL:%[0-9]+]]:_(s32) = G_PHI [[OLDVALSTART]](s32), %bb.1, [[OLDVALRES:%[0-9]+]](s32), %bb.2 +; CHECK-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[OLDVAL]], [[VAL]] +; CHECK-NEXT: [[NEWVAL:%[0-9]+]]:_(s32) = G_XOR [[AND]], [[NEG1]] +; CHECK: [[OLDVALRES]]:_(s32), [[SUCCESS:%[0-9]+]]:_(s1) = G_ATOMIC_CMPXCHG_WITH_SUCCESS [[ADDR]](p0), [[OLDVAL]], [[NEWVAL]] :: (load store seq_cst seq_cst (s32) on %ir.addr) +; CHECK-NEXT: G_BRCOND [[SUCCESS]](s1), %bb.3 +; CHECK-NEXT: G_BR %bb.2 +; CHECK: bb.3.atomicrmw.end: + %oldval = atomicrmw nand ptr %addr, i32 1 seq_cst + ret i32 %oldval } ; Try an seq_cst atomicrmw or -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_or(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_or ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_OR [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw or ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_OR [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s32) on %ir.addr) + %oldval = atomicrmw or ptr %addr, i32 1 seq_cst + ret i32 %oldval } ; Try an seq_cst atomicrmw xor -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_xor(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_xor ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_XOR [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw xor ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_XOR [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s32) on %ir.addr) + %oldval = atomicrmw xor ptr %addr, i32 1 seq_cst + ret i32 %oldval } ; Try an seq_cst atomicrmw min -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_min(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_min ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_MIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw min ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_MIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s32) on %ir.addr) + %oldval = atomicrmw min ptr %addr, i32 1 seq_cst + ret i32 %oldval } ; Try an seq_cst atomicrmw max -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_max(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_max ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_MAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw max ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_MAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s32) on %ir.addr) + %oldval = atomicrmw max ptr %addr, i32 1 seq_cst + ret i32 %oldval } ; Try an seq_cst atomicrmw unsigned min -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_umin(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_umin ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_UMIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw umin ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_UMIN [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s32) on %ir.addr) + %oldval = atomicrmw umin ptr %addr, i32 1 seq_cst + ret i32 %oldval } ; Try an seq_cst atomicrmw unsigned max -; AArch64 will expand some atomicrmw's at the LLVM-IR level so we use a wide type to avoid this. define i32 @test_atomicrmw_umax(ptr %addr) { ; CHECK-LABEL: name: test_atomicrmw_umax ; CHECK: bb.1 (%ir-block.{{[0-9]+}}): ; CHECK-NEXT: liveins: $x0 ; CHECK: [[ADDR:%[0-9]+]]:_(p0) = COPY $x0 -; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s256) = G_CONSTANT i256 1 -; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s256) = G_ATOMICRMW_UMAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s256) on %ir.addr) -; CHECK-NEXT: [[RES:%[0-9]+]]:_(s32) = G_TRUNC [[OLDVALRES]] - %oldval = atomicrmw umax ptr %addr, i256 1 seq_cst - ; FIXME: We currently can't lower 'ret i256' and it's not the purpose of this - ; test so work around it by truncating to i32 for now. - %oldval.trunc = trunc i256 %oldval to i32 - ret i32 %oldval.trunc +; CHECK-NEXT: [[VAL:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 +; CHECK-NEXT: [[OLDVALRES:%[0-9]+]]:_(s32) = G_ATOMICRMW_UMAX [[ADDR]](p0), [[VAL]] :: (load store seq_cst (s32) on %ir.addr) + %oldval = atomicrmw umax ptr %addr, i32 1 seq_cst + ret i32 %oldval } @addr = global ptr null diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir index ae15e74a4327..178db852e35b 100644 --- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir +++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir @@ -258,6 +258,9 @@ # DEBUG-NEXT: G_FENCE (opcode {{[0-9]+}}): 0 type indices # DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined # DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined +# DEBUG-NEXT: G_PREFETCH (opcode {{[0-9]+}}): 1 type index, 0 imm indices +# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected +# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected # DEBUG-NEXT: G_BRCOND (opcode {{[0-9]+}}): 1 type index, 0 imm indices # DEBUG-NEXT: .. the first uncovered type index: 1, OK # DEBUG-NEXT: .. the first uncovered imm index: 0, OK diff --git a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll index 55750ab34e17..3c8aca514526 100644 --- a/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll +++ b/llvm/test/CodeGen/AArch64/aarch64-neon-vector-insert-uaddlv.ll @@ -69,11 +69,10 @@ define void @insert_vec_v23i32_uaddlv_from_v8i16(ptr %0) { ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: movi.2d v0, #0000000000000000 ; CHECK-NEXT: movi.2d v2, #0000000000000000 -; CHECK-NEXT: add x8, x0, #88 +; CHECK-NEXT: str wzr, [x0, #88] ; CHECK-NEXT: uaddlv.8h s1, v0 ; CHECK-NEXT: stp q0, q0, [x0, #16] ; CHECK-NEXT: stp q0, q0, [x0, #48] -; CHECK-NEXT: st1.s { v0 }[2], [x8] ; CHECK-NEXT: str d0, [x0, #80] ; CHECK-NEXT: mov.s v2[0], v1[0] ; CHECK-NEXT: ucvtf.4s v1, v2 diff --git a/llvm/test/CodeGen/AArch64/arm64-promote-const-complex-initializers.ll b/llvm/test/CodeGen/AArch64/arm64-promote-const-complex-initializers.ll index 86ebf803c578..bbd2acbab424 100644 --- a/llvm/test/CodeGen/AArch64/arm64-promote-const-complex-initializers.ll +++ b/llvm/test/CodeGen/AArch64/arm64-promote-const-complex-initializers.ll @@ -30,10 +30,10 @@ define [1 x <4 x float>] @test1() { define [1 x <4 x float>] @test2() { ; CHECK-LABEL: .p2align 4, 0x0 ; -- Begin function test2 ; CHECK-NEXT: lCPI1_0: -; CHECK-NEXT: .long 0x00000000 ; float 0 -; CHECK-NEXT: .long 0x00000000 ; float 0 -; CHECK-NEXT: .long 0x00000000 ; float 0 -; CHECK-NEXT: .long 0x3f800000 ; float 1 +; CHECK-NEXT: .long 0x80000000 ; float -0 +; CHECK-NEXT: .long 0x80000000 ; float -0 +; CHECK-NEXT: .long 0x80000000 ; float -0 +; CHECK-NEXT: .long 0xbf800000 ; float -1 ; CHECK-NEXT: .section __TEXT,__text,regular,pure_instructions ; CHECK-NEXT: .globl _test2 ; CHECK-NEXT: .p2align 2 @@ -43,17 +43,7 @@ define [1 x <4 x float>] @test2() { ; CHECK-NEXT: Lloh2: ; CHECK-NEXT: adrp x8, lCPI1_0@PAGE ; CHECK-NEXT: Lloh3: -; CHECK-NEXT: ldr q1, [x8, lCPI1_0@PAGEOFF] -; CHECK-NEXT: mov s2, v1[1] -; CHECK-NEXT: fneg s0, s1 -; CHECK-NEXT: mov s3, v1[2] -; CHECK-NEXT: mov s1, v1[3] -; CHECK-NEXT: fneg s2, s2 -; CHECK-NEXT: fneg s3, s3 -; CHECK-NEXT: fneg s1, s1 -; CHECK-NEXT: mov.s v0[1], v2[0] -; CHECK-NEXT: mov.s v0[2], v3[0] -; CHECK-NEXT: mov.s v0[3], v1[0] +; CHECK-NEXT: ldr q0, [x8, lCPI1_0@PAGEOFF] ; CHECK-NEXT: ret ; %constexpr = fneg float extractelement (<4 x float> bitcast (<1 x i128> to <4 x float>), i32 0) diff --git a/llvm/test/CodeGen/AArch64/atomic-oversize.ll b/llvm/test/CodeGen/AArch64/atomic-oversize.ll new file mode 100644 index 000000000000..9065311a9aaa --- /dev/null +++ b/llvm/test/CodeGen/AArch64/atomic-oversize.ll @@ -0,0 +1,11 @@ +; RUN: llc -march=aarch64 < %s | FileCheck %s + +; Atomics larger than 128-bit are unsupported, and emit libcalls. +define void @test(ptr %a) nounwind { +; CHECK-LABEL: test: +; CHECK: bl __atomic_load +; CHECK: bl __atomic_store + %1 = load atomic i256, ptr %a seq_cst, align 32 + store atomic i256 %1, ptr %a seq_cst, align 32 + ret void +} diff --git a/llvm/test/CodeGen/AArch64/cmp-select-sign.ll b/llvm/test/CodeGen/AArch64/cmp-select-sign.ll index d16b5786a996..09a6e26fe5a4 100644 --- a/llvm/test/CodeGen/AArch64/cmp-select-sign.ll +++ b/llvm/test/CodeGen/AArch64/cmp-select-sign.ll @@ -176,12 +176,11 @@ define <4 x i32> @sign_4xi32_multi_use(<4 x i32> %a) { ; CHECK-NEXT: str x30, [sp, #16] // 8-byte Folded Spill ; CHECK-NEXT: .cfi_def_cfa_offset 32 ; CHECK-NEXT: .cfi_offset w30, -16 -; CHECK-NEXT: movi v1.2d, #0xffffffffffffffff -; CHECK-NEXT: cmlt v2.4s, v0.4s, #0 -; CHECK-NEXT: orr v2.4s, #1 -; CHECK-NEXT: cmgt v1.4s, v0.4s, v1.4s -; CHECK-NEXT: str q2, [sp] // 16-byte Folded Spill -; CHECK-NEXT: xtn v0.4h, v1.4s +; CHECK-NEXT: cmlt v1.4s, v0.4s, #0 +; CHECK-NEXT: cmge v0.4s, v0.4s, #0 +; CHECK-NEXT: xtn v0.4h, v0.4s +; CHECK-NEXT: orr v1.4s, #1 +; CHECK-NEXT: str q1, [sp] // 16-byte Folded Spill ; CHECK-NEXT: bl use_4xi1 ; CHECK-NEXT: ldr q0, [sp] // 16-byte Folded Reload ; CHECK-NEXT: ldr x30, [sp, #16] // 8-byte Folded Reload @@ -227,9 +226,8 @@ define <4 x i32> @not_sign_4xi32_2(<4 x i32> %a) { define <4 x i32> @not_sign_4xi32_3(<4 x i32> %a) { ; CHECK-LABEL: not_sign_4xi32_3: ; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0xffffffffffffffff ; CHECK-NEXT: adrp x8, .LCPI18_0 -; CHECK-NEXT: cmgt v0.4s, v0.4s, v1.4s +; CHECK-NEXT: cmge v0.4s, v0.4s, #0 ; CHECK-NEXT: ldr q1, [x8, :lo12:.LCPI18_0] ; CHECK-NEXT: bic v1.16b, v1.16b, v0.16b ; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s diff --git a/llvm/test/CodeGen/AArch64/dag-numsignbits.ll b/llvm/test/CodeGen/AArch64/dag-numsignbits.ll index 575102ce4ff9..11f19b8c5696 100644 --- a/llvm/test/CodeGen/AArch64/dag-numsignbits.ll +++ b/llvm/test/CodeGen/AArch64/dag-numsignbits.ll @@ -8,13 +8,12 @@ define void @signbits_vXi1(<4 x i16> %a1) { ; CHECK: // %bb.0: ; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0 ; CHECK-NEXT: adrp x8, .LCPI0_0 -; CHECK-NEXT: movi v2.4h, #1 ; CHECK-NEXT: mov w1, wzr +; CHECK-NEXT: mov w2, wzr ; CHECK-NEXT: dup v0.4h, v0.h[0] ; CHECK-NEXT: ldr d1, [x8, :lo12:.LCPI0_0] -; CHECK-NEXT: mov w2, wzr ; CHECK-NEXT: add v0.4h, v0.4h, v1.4h -; CHECK-NEXT: cmgt v0.4h, v2.4h, v0.4h +; CHECK-NEXT: cmle v0.4h, v0.4h, #0 ; CHECK-NEXT: umov w0, v0.h[0] ; CHECK-NEXT: umov w3, v0.h[3] ; CHECK-NEXT: b foo diff --git a/llvm/test/CodeGen/AArch64/fpenv.ll b/llvm/test/CodeGen/AArch64/fpenv.ll index 14bf7888918e..3a307f773103 100644 --- a/llvm/test/CodeGen/AArch64/fpenv.ll +++ b/llvm/test/CodeGen/AArch64/fpenv.ll @@ -1,70 +1,65 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s -define void @func_02(i32 %rm) { +define void @func_set_rounding_dyn(i32 %rm) { +; CHECK-LABEL: func_set_rounding_dyn: +; CHECK: // %bb.0: +; CHECK-NEXT: sub w9, w0, #1 +; CHECK-NEXT: mrs x8, FPCR +; CHECK-NEXT: and w9, w9, #0x3 +; CHECK-NEXT: and x8, x8, #0xffffffffff3fffff +; CHECK-NEXT: lsl w9, w9, #22 +; CHECK-NEXT: orr x8, x8, x9 +; CHECK-NEXT: msr FPCR, x8 +; CHECK-NEXT: ret call void @llvm.set.rounding(i32 %rm) ret void } -; CHECK-LABEL: func_02: -; CHECK: sub w9, w0, #1 -; CHECK: mrs x8, FPCR -; CHECK: and w9, w9, #0x3 -; CHECK: and x8, x8, #0xffffffffff3fffff -; CHECK: lsl w9, w9, #22 -; CHECK: orr x8, x8, x9 -; CHECK: msr FPCR, x8 -; CHECK: ret - - -define void @func_03() { +define void @func_set_rounding_toward_zero() { +; CHECK-LABEL: func_set_rounding_toward_zero: +; CHECK: // %bb.0: +; CHECK-NEXT: mrs x8, FPCR +; CHECK-NEXT: orr x8, x8, #0xc00000 +; CHECK-NEXT: msr FPCR, x8 +; CHECK-NEXT: ret call void @llvm.set.rounding(i32 0) ret void } -; CHECK-LABEL: func_03 -; CHECK: mrs x8, FPCR -; CHECK: orr x8, x8, #0xc00000 -; CHECK: msr FPCR, x8 -; CHECK: ret - - -define void @func_04() { +define void @func_set_rounding_tonearest_even() { +; CHECK-LABEL: func_set_rounding_tonearest_even: +; CHECK: // %bb.0: +; CHECK-NEXT: mrs x8, FPCR +; CHECK-NEXT: and x8, x8, #0xffffffffff3fffff +; CHECK-NEXT: msr FPCR, x8 +; CHECK-NEXT: ret call void @llvm.set.rounding(i32 1) ret void } -; CHECK-LABEL: func_04 -; CHECK: mrs x8, FPCR -; CHECK: and x8, x8, #0xffffffffff3fffff -; CHECK: msr FPCR, x8 -; CHECK: ret - - -define void @func_05() { +define void @func_set_rounding_upward() { +; CHECK-LABEL: func_set_rounding_upward: +; CHECK: // %bb.0: +; CHECK-NEXT: mrs x8, FPCR +; CHECK-NEXT: and x8, x8, #0xffffffffff3fffff +; CHECK-NEXT: orr x8, x8, #0x400000 +; CHECK-NEXT: msr FPCR, x8 +; CHECK-NEXT: ret call void @llvm.set.rounding(i32 2) ret void } - -; CHECK-LABEL: func_05 -; CHECK: mrs x8, FPCR -; CHECK: and x8, x8, #0xffffffffff3fffff -; CHECK: orr x8, x8, #0x400000 -; CHECK: msr FPCR, x8 -; CHECK: ret - - -define void @func_06() { +define void @func_set_rounding_downward() { +; CHECK-LABEL: func_set_rounding_downward: +; CHECK: // %bb.0: +; CHECK-NEXT: mrs x8, FPCR +; CHECK-NEXT: and x8, x8, #0xffffffffff3fffff +; CHECK-NEXT: orr x8, x8, #0x800000 +; CHECK-NEXT: msr FPCR, x8 +; CHECK-NEXT: ret call void @llvm.set.rounding(i32 3) ret void } -; CHECK-LABEL: func_06 -; CHECK: mrs x8, FPCR -; CHECK: and x8, x8, #0xffffffffff3fffff -; CHECK: orr x8, x8, #0x800000 -; CHECK: msr FPCR, x8 -; CHECK: ret - - declare void @llvm.set.rounding(i32) diff --git a/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll b/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll index 6ee3cb489285..ee3e808f9f92 100644 --- a/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll +++ b/llvm/test/CodeGen/AArch64/misched-fusion-aes.ll @@ -12,7 +12,6 @@ ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=cortex-x1 | FileCheck %s ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=neoverse-e1 | FileCheck %s ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=neoverse-n1 | FileCheck %s -; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=neoverse-n2 | FileCheck %s ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=neoverse-v1 | FileCheck %s ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=neoverse-512tvb | FileCheck %s ; RUN: llc %s -o - -mtriple=aarch64-unknown -mcpu=exynos-m3 | FileCheck %s diff --git a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll index 0abdda5954f5..2d594947682b 100644 --- a/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll +++ b/llvm/test/CodeGen/AArch64/neon-compare-instructions.ll @@ -7,7 +7,7 @@ define <8 x i8> @cmeq8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret - %tmp3 = icmp eq <8 x i8> %A, %B; + %tmp3 = icmp eq <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -17,7 +17,7 @@ define <16 x i8> @cmeq16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret - %tmp3 = icmp eq <16 x i8> %A, %B; + %tmp3 = icmp eq <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -27,7 +27,7 @@ define <4 x i16> @cmeq4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.4h, v0.4h, v1.4h ; CHECK-NEXT: ret - %tmp3 = icmp eq <4 x i16> %A, %B; + %tmp3 = icmp eq <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -37,7 +37,7 @@ define <8 x i16> @cmeq8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.8h, v0.8h, v1.8h ; CHECK-NEXT: ret - %tmp3 = icmp eq <8 x i16> %A, %B; + %tmp3 = icmp eq <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -47,7 +47,7 @@ define <2 x i32> @cmeq2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.2s, v0.2s, v1.2s ; CHECK-NEXT: ret - %tmp3 = icmp eq <2 x i32> %A, %B; + %tmp3 = icmp eq <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -57,7 +57,7 @@ define <4 x i32> @cmeq4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret - %tmp3 = icmp eq <4 x i32> %A, %B; + %tmp3 = icmp eq <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -67,7 +67,7 @@ define <2 x i64> @cmeq2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmeq v0.2d, v0.2d, v1.2d ; CHECK-NEXT: ret - %tmp3 = icmp eq <2 x i64> %A, %B; + %tmp3 = icmp eq <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -78,7 +78,7 @@ define <8 x i8> @cmne8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK-NEXT: cmeq v0.8b, v0.8b, v1.8b ; CHECK-NEXT: mvn v0.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp ne <8 x i8> %A, %B; + %tmp3 = icmp ne <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -89,7 +89,7 @@ define <16 x i8> @cmne16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK-NEXT: cmeq v0.16b, v0.16b, v1.16b ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ne <16 x i8> %A, %B; + %tmp3 = icmp ne <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -100,7 +100,7 @@ define <4 x i16> @cmne4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK-NEXT: cmeq v0.4h, v0.4h, v1.4h ; CHECK-NEXT: mvn v0.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp ne <4 x i16> %A, %B; + %tmp3 = icmp ne <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -111,7 +111,7 @@ define <8 x i16> @cmne8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK-NEXT: cmeq v0.8h, v0.8h, v1.8h ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ne <8 x i16> %A, %B; + %tmp3 = icmp ne <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -122,7 +122,7 @@ define <2 x i32> @cmne2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK-NEXT: cmeq v0.2s, v0.2s, v1.2s ; CHECK-NEXT: mvn v0.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp ne <2 x i32> %A, %B; + %tmp3 = icmp ne <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -133,7 +133,7 @@ define <4 x i32> @cmne4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK-NEXT: cmeq v0.4s, v0.4s, v1.4s ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ne <4 x i32> %A, %B; + %tmp3 = icmp ne <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -144,7 +144,7 @@ define <2 x i64> @cmne2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK-NEXT: cmeq v0.2d, v0.2d, v1.2d ; CHECK-NEXT: mvn v0.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ne <2 x i64> %A, %B; + %tmp3 = icmp ne <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -154,7 +154,7 @@ define <8 x i8> @cmgt8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret - %tmp3 = icmp sgt <8 x i8> %A, %B; + %tmp3 = icmp sgt <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -164,7 +164,7 @@ define <16 x i8> @cmgt16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret - %tmp3 = icmp sgt <16 x i8> %A, %B; + %tmp3 = icmp sgt <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -174,7 +174,7 @@ define <4 x i16> @cmgt4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.4h, v0.4h, v1.4h ; CHECK-NEXT: ret - %tmp3 = icmp sgt <4 x i16> %A, %B; + %tmp3 = icmp sgt <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -184,7 +184,7 @@ define <8 x i16> @cmgt8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.8h, v0.8h, v1.8h ; CHECK-NEXT: ret - %tmp3 = icmp sgt <8 x i16> %A, %B; + %tmp3 = icmp sgt <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -194,7 +194,7 @@ define <2 x i32> @cmgt2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.2s, v0.2s, v1.2s ; CHECK-NEXT: ret - %tmp3 = icmp sgt <2 x i32> %A, %B; + %tmp3 = icmp sgt <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -204,7 +204,7 @@ define <4 x i32> @cmgt4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret - %tmp3 = icmp sgt <4 x i32> %A, %B; + %tmp3 = icmp sgt <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -214,7 +214,7 @@ define <2 x i64> @cmgt2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.2d, v0.2d, v1.2d ; CHECK-NEXT: ret - %tmp3 = icmp sgt <2 x i64> %A, %B; + %tmp3 = icmp sgt <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -225,7 +225,7 @@ define <8 x i8> @cmlt8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp slt <8 x i8> %A, %B; + %tmp3 = icmp slt <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -236,7 +236,7 @@ define <16 x i8> @cmlt16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp slt <16 x i8> %A, %B; + %tmp3 = icmp slt <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -247,7 +247,7 @@ define <4 x i16> @cmlt4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.4h, v1.4h, v0.4h ; CHECK-NEXT: ret - %tmp3 = icmp slt <4 x i16> %A, %B; + %tmp3 = icmp slt <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -258,7 +258,7 @@ define <8 x i16> @cmlt8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.8h, v1.8h, v0.8h ; CHECK-NEXT: ret - %tmp3 = icmp slt <8 x i16> %A, %B; + %tmp3 = icmp slt <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -269,7 +269,7 @@ define <2 x i32> @cmlt2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.2s, v1.2s, v0.2s ; CHECK-NEXT: ret - %tmp3 = icmp slt <2 x i32> %A, %B; + %tmp3 = icmp slt <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -280,7 +280,7 @@ define <4 x i32> @cmlt4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret - %tmp3 = icmp slt <4 x i32> %A, %B; + %tmp3 = icmp slt <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -291,7 +291,7 @@ define <2 x i64> @cmlt2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmgt v0.2d, v1.2d, v0.2d ; CHECK-NEXT: ret - %tmp3 = icmp slt <2 x i64> %A, %B; + %tmp3 = icmp slt <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -301,7 +301,7 @@ define <8 x i8> @cmge8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret - %tmp3 = icmp sge <8 x i8> %A, %B; + %tmp3 = icmp sge <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -311,7 +311,7 @@ define <16 x i8> @cmge16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret - %tmp3 = icmp sge <16 x i8> %A, %B; + %tmp3 = icmp sge <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -321,7 +321,7 @@ define <4 x i16> @cmge4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.4h, v0.4h, v1.4h ; CHECK-NEXT: ret - %tmp3 = icmp sge <4 x i16> %A, %B; + %tmp3 = icmp sge <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -331,7 +331,7 @@ define <8 x i16> @cmge8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.8h, v0.8h, v1.8h ; CHECK-NEXT: ret - %tmp3 = icmp sge <8 x i16> %A, %B; + %tmp3 = icmp sge <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -341,7 +341,7 @@ define <2 x i32> @cmge2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.2s, v0.2s, v1.2s ; CHECK-NEXT: ret - %tmp3 = icmp sge <2 x i32> %A, %B; + %tmp3 = icmp sge <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -351,7 +351,7 @@ define <4 x i32> @cmge4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret - %tmp3 = icmp sge <4 x i32> %A, %B; + %tmp3 = icmp sge <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -361,7 +361,7 @@ define <2 x i64> @cmge2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.2d, v0.2d, v1.2d ; CHECK-NEXT: ret - %tmp3 = icmp sge <2 x i64> %A, %B; + %tmp3 = icmp sge <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -372,7 +372,7 @@ define <8 x i8> @cmle8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp sle <8 x i8> %A, %B; + %tmp3 = icmp sle <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -383,7 +383,7 @@ define <16 x i8> @cmle16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp sle <16 x i8> %A, %B; + %tmp3 = icmp sle <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -394,7 +394,7 @@ define <4 x i16> @cmle4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.4h, v1.4h, v0.4h ; CHECK-NEXT: ret - %tmp3 = icmp sle <4 x i16> %A, %B; + %tmp3 = icmp sle <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -405,7 +405,7 @@ define <8 x i16> @cmle8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.8h, v1.8h, v0.8h ; CHECK-NEXT: ret - %tmp3 = icmp sle <8 x i16> %A, %B; + %tmp3 = icmp sle <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -416,7 +416,7 @@ define <2 x i32> @cmle2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.2s, v1.2s, v0.2s ; CHECK-NEXT: ret - %tmp3 = icmp sle <2 x i32> %A, %B; + %tmp3 = icmp sle <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -427,7 +427,7 @@ define <4 x i32> @cmle4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret - %tmp3 = icmp sle <4 x i32> %A, %B; + %tmp3 = icmp sle <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -438,7 +438,7 @@ define <2 x i64> @cmle2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmge v0.2d, v1.2d, v0.2d ; CHECK-NEXT: ret - %tmp3 = icmp sle <2 x i64> %A, %B; + %tmp3 = icmp sle <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -448,7 +448,7 @@ define <8 x i8> @cmhi8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret - %tmp3 = icmp ugt <8 x i8> %A, %B; + %tmp3 = icmp ugt <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -458,7 +458,7 @@ define <16 x i8> @cmhi16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret - %tmp3 = icmp ugt <16 x i8> %A, %B; + %tmp3 = icmp ugt <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -468,7 +468,7 @@ define <4 x i16> @cmhi4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.4h, v0.4h, v1.4h ; CHECK-NEXT: ret - %tmp3 = icmp ugt <4 x i16> %A, %B; + %tmp3 = icmp ugt <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -478,7 +478,7 @@ define <8 x i16> @cmhi8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.8h, v0.8h, v1.8h ; CHECK-NEXT: ret - %tmp3 = icmp ugt <8 x i16> %A, %B; + %tmp3 = icmp ugt <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -488,7 +488,7 @@ define <2 x i32> @cmhi2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.2s, v0.2s, v1.2s ; CHECK-NEXT: ret - %tmp3 = icmp ugt <2 x i32> %A, %B; + %tmp3 = icmp ugt <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -498,7 +498,7 @@ define <4 x i32> @cmhi4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret - %tmp3 = icmp ugt <4 x i32> %A, %B; + %tmp3 = icmp ugt <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -508,7 +508,7 @@ define <2 x i64> @cmhi2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.2d, v0.2d, v1.2d ; CHECK-NEXT: ret - %tmp3 = icmp ugt <2 x i64> %A, %B; + %tmp3 = icmp ugt <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -519,7 +519,7 @@ define <8 x i8> @cmlo8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp ult <8 x i8> %A, %B; + %tmp3 = icmp ult <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -530,7 +530,7 @@ define <16 x i8> @cmlo16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ult <16 x i8> %A, %B; + %tmp3 = icmp ult <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -541,7 +541,7 @@ define <4 x i16> @cmlo4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.4h, v1.4h, v0.4h ; CHECK-NEXT: ret - %tmp3 = icmp ult <4 x i16> %A, %B; + %tmp3 = icmp ult <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -552,7 +552,7 @@ define <8 x i16> @cmlo8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.8h, v1.8h, v0.8h ; CHECK-NEXT: ret - %tmp3 = icmp ult <8 x i16> %A, %B; + %tmp3 = icmp ult <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -563,7 +563,7 @@ define <2 x i32> @cmlo2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.2s, v1.2s, v0.2s ; CHECK-NEXT: ret - %tmp3 = icmp ult <2 x i32> %A, %B; + %tmp3 = icmp ult <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -574,7 +574,7 @@ define <4 x i32> @cmlo4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret - %tmp3 = icmp ult <4 x i32> %A, %B; + %tmp3 = icmp ult <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -585,7 +585,7 @@ define <2 x i64> @cmlo2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhi v0.2d, v1.2d, v0.2d ; CHECK-NEXT: ret - %tmp3 = icmp ult <2 x i64> %A, %B; + %tmp3 = icmp ult <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -595,7 +595,7 @@ define <8 x i8> @cmhs8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.8b, v0.8b, v1.8b ; CHECK-NEXT: ret - %tmp3 = icmp uge <8 x i8> %A, %B; + %tmp3 = icmp uge <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -605,7 +605,7 @@ define <16 x i8> @cmhs16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.16b, v0.16b, v1.16b ; CHECK-NEXT: ret - %tmp3 = icmp uge <16 x i8> %A, %B; + %tmp3 = icmp uge <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -615,7 +615,7 @@ define <4 x i16> @cmhs4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.4h, v0.4h, v1.4h ; CHECK-NEXT: ret - %tmp3 = icmp uge <4 x i16> %A, %B; + %tmp3 = icmp uge <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -625,7 +625,7 @@ define <8 x i16> @cmhs8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.8h, v0.8h, v1.8h ; CHECK-NEXT: ret - %tmp3 = icmp uge <8 x i16> %A, %B; + %tmp3 = icmp uge <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -635,7 +635,7 @@ define <2 x i32> @cmhs2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.2s, v0.2s, v1.2s ; CHECK-NEXT: ret - %tmp3 = icmp uge <2 x i32> %A, %B; + %tmp3 = icmp uge <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -645,7 +645,7 @@ define <4 x i32> @cmhs4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret - %tmp3 = icmp uge <4 x i32> %A, %B; + %tmp3 = icmp uge <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -655,7 +655,7 @@ define <2 x i64> @cmhs2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.2d, v0.2d, v1.2d ; CHECK-NEXT: ret - %tmp3 = icmp uge <2 x i64> %A, %B; + %tmp3 = icmp uge <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -666,7 +666,7 @@ define <8 x i8> @cmls8xi8(<8 x i8> %A, <8 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp ule <8 x i8> %A, %B; + %tmp3 = icmp ule <8 x i8> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -677,7 +677,7 @@ define <16 x i8> @cmls16xi8(<16 x i8> %A, <16 x i8> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ule <16 x i8> %A, %B; + %tmp3 = icmp ule <16 x i8> %A, %B %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -688,7 +688,7 @@ define <4 x i16> @cmls4xi16(<4 x i16> %A, <4 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.4h, v1.4h, v0.4h ; CHECK-NEXT: ret - %tmp3 = icmp ule <4 x i16> %A, %B; + %tmp3 = icmp ule <4 x i16> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -699,7 +699,7 @@ define <8 x i16> @cmls8xi16(<8 x i16> %A, <8 x i16> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.8h, v1.8h, v0.8h ; CHECK-NEXT: ret - %tmp3 = icmp ule <8 x i16> %A, %B; + %tmp3 = icmp ule <8 x i16> %A, %B %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -710,7 +710,7 @@ define <2 x i32> @cmls2xi32(<2 x i32> %A, <2 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.2s, v1.2s, v0.2s ; CHECK-NEXT: ret - %tmp3 = icmp ule <2 x i32> %A, %B; + %tmp3 = icmp ule <2 x i32> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -721,7 +721,7 @@ define <4 x i32> @cmls4xi32(<4 x i32> %A, <4 x i32> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret - %tmp3 = icmp ule <4 x i32> %A, %B; + %tmp3 = icmp ule <4 x i32> %A, %B %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -732,7 +732,7 @@ define <2 x i64> @cmls2xi64(<2 x i64> %A, <2 x i64> %B) { ; CHECK: // %bb.0: ; CHECK-NEXT: cmhs v0.2d, v1.2d, v0.2d ; CHECK-NEXT: ret - %tmp3 = icmp ule <2 x i64> %A, %B; + %tmp3 = icmp ule <2 x i64> %A, %B %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -883,7 +883,7 @@ define <8 x i8> @cmeqz8xi8(<8 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.8b, v0.8b, v1.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <8 x i8> %A, zeroinitializer; + %tmp3 = icmp eq <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -899,7 +899,7 @@ define <16 x i8> @cmeqz16xi8(<16 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <16 x i8> %A, zeroinitializer; + %tmp3 = icmp eq <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -915,7 +915,7 @@ define <4 x i16> @cmeqz4xi16(<4 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.4h, v0.4h, v1.4h ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <4 x i16> %A, zeroinitializer; + %tmp3 = icmp eq <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -931,7 +931,7 @@ define <8 x i16> @cmeqz8xi16(<8 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.8h, v0.8h, v1.8h ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <8 x i16> %A, zeroinitializer; + %tmp3 = icmp eq <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -947,7 +947,7 @@ define <2 x i32> @cmeqz2xi32(<2 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.2s, v0.2s, v1.2s ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <2 x i32> %A, zeroinitializer; + %tmp3 = icmp eq <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -963,7 +963,7 @@ define <4 x i32> @cmeqz4xi32(<4 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <4 x i32> %A, zeroinitializer; + %tmp3 = icmp eq <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -979,7 +979,7 @@ define <2 x i64> @cmeqz2xi64(<2 x i64> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmeq v0.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: ret - %tmp3 = icmp eq <2 x i64> %A, zeroinitializer; + %tmp3 = icmp eq <2 x i64> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -996,7 +996,7 @@ define <8 x i8> @cmgez8xi8(<8 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.8b, v0.8b, v1.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <8 x i8> %A, zeroinitializer; + %tmp3 = icmp sge <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -1012,7 +1012,7 @@ define <16 x i8> @cmgez16xi8(<16 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <16 x i8> %A, zeroinitializer; + %tmp3 = icmp sge <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -1028,7 +1028,7 @@ define <4 x i16> @cmgez4xi16(<4 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.4h, v0.4h, v1.4h ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <4 x i16> %A, zeroinitializer; + %tmp3 = icmp sge <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -1044,7 +1044,7 @@ define <8 x i16> @cmgez8xi16(<8 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.8h, v0.8h, v1.8h ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <8 x i16> %A, zeroinitializer; + %tmp3 = icmp sge <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -1060,7 +1060,7 @@ define <2 x i32> @cmgez2xi32(<2 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.2s, v0.2s, v1.2s ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <2 x i32> %A, zeroinitializer; + %tmp3 = icmp sge <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -1076,7 +1076,7 @@ define <4 x i32> @cmgez4xi32(<4 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <4 x i32> %A, zeroinitializer; + %tmp3 = icmp sge <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -1092,7 +1092,7 @@ define <2 x i64> @cmgez2xi64(<2 x i64> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: ret - %tmp3 = icmp sge <2 x i64> %A, zeroinitializer; + %tmp3 = icmp sge <2 x i64> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -1210,6 +1210,118 @@ define <2 x i64> @cmgez2xi64_alt(<2 x i64> %A) { ret <2 x i64> %not } +define <8 x i8> @cmgez8xi8_alt2(<8 x i8> %A) { +; CHECK-SD-LABEL: cmgez8xi8_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.8b, v0.8b, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez8xi8_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi d1, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.8b, v0.8b, v1.8b +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <8 x i8> %A, + %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> + ret <8 x i8> %tmp4 +} + +define <16 x i8> @cmgez16xi8_alt2(<16 x i8> %A) { +; CHECK-SD-LABEL: cmgez16xi8_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.16b, v0.16b, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez16xi8_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.2d, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.16b, v0.16b, v1.16b +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <16 x i8> %A, + %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> + ret <16 x i8> %tmp4 +} + +define <4 x i16> @cmgez4xi16_alt2(<4 x i16> %A) { +; CHECK-SD-LABEL: cmgez4xi16_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.4h, v0.4h, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez4xi16_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi d1, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.4h, v0.4h, v1.4h +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <4 x i16> %A, + %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> + ret <4 x i16> %tmp4 +} + +define <8 x i16> @cmgez8xi16_alt2(<8 x i16> %A) { +; CHECK-SD-LABEL: cmgez8xi16_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.8h, v0.8h, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez8xi16_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.2d, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.8h, v0.8h, v1.8h +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <8 x i16> %A, + %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> + ret <8 x i16> %tmp4 +} + +define <2 x i32> @cmgez2xi32_alt2(<2 x i32> %A) { +; CHECK-SD-LABEL: cmgez2xi32_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.2s, v0.2s, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez2xi32_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi d1, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.2s, v0.2s, v1.2s +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <2 x i32> %A, + %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> + ret <2 x i32> %tmp4 +} + +define <4 x i32> @cmgez4xi32_alt2(<4 x i32> %A) { +; CHECK-SD-LABEL: cmgez4xi32_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.4s, v0.4s, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez4xi32_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.2d, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.4s, v0.4s, v1.4s +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <4 x i32> %A, + %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> + ret <4 x i32> %tmp4 +} + +define <2 x i64> @cmgez2xi64_alt2(<2 x i64> %A) { +; CHECK-SD-LABEL: cmgez2xi64_alt2: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmge v0.2d, v0.2d, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmgez2xi64_alt2: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.2d, #0xffffffffffffffff +; CHECK-GI-NEXT: cmgt v0.2d, v0.2d, v1.2d +; CHECK-GI-NEXT: ret + %tmp3 = icmp sgt <2 x i64> %A, + %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> + ret <2 x i64> %tmp4 +} + define <8 x i8> @cmgtz8xi8(<8 x i8> %A) { ; CHECK-SD-LABEL: cmgtz8xi8: @@ -1222,7 +1334,7 @@ define <8 x i8> @cmgtz8xi8(<8 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.8b, v0.8b, v1.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <8 x i8> %A, zeroinitializer; + %tmp3 = icmp sgt <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -1238,7 +1350,7 @@ define <16 x i8> @cmgtz16xi8(<16 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <16 x i8> %A, zeroinitializer; + %tmp3 = icmp sgt <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -1254,7 +1366,7 @@ define <4 x i16> @cmgtz4xi16(<4 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.4h, v0.4h, v1.4h ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <4 x i16> %A, zeroinitializer; + %tmp3 = icmp sgt <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -1270,7 +1382,7 @@ define <8 x i16> @cmgtz8xi16(<8 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.8h, v0.8h, v1.8h ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <8 x i16> %A, zeroinitializer; + %tmp3 = icmp sgt <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -1286,7 +1398,7 @@ define <2 x i32> @cmgtz2xi32(<2 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.2s, v0.2s, v1.2s ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <2 x i32> %A, zeroinitializer; + %tmp3 = icmp sgt <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -1302,7 +1414,7 @@ define <4 x i32> @cmgtz4xi32(<4 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <4 x i32> %A, zeroinitializer; + %tmp3 = icmp sgt <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -1318,7 +1430,7 @@ define <2 x i64> @cmgtz2xi64(<2 x i64> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: ret - %tmp3 = icmp sgt <2 x i64> %A, zeroinitializer; + %tmp3 = icmp sgt <2 x i64> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -1334,7 +1446,7 @@ define <8 x i8> @cmlez8xi8(<8 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.8b, v1.8b, v0.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <8 x i8> %A, zeroinitializer; + %tmp3 = icmp sle <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -1350,7 +1462,7 @@ define <16 x i8> @cmlez16xi8(<16 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.16b, v1.16b, v0.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <16 x i8> %A, zeroinitializer; + %tmp3 = icmp sle <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -1366,7 +1478,7 @@ define <4 x i16> @cmlez4xi16(<4 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.4h, v1.4h, v0.4h ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <4 x i16> %A, zeroinitializer; + %tmp3 = icmp sle <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -1382,7 +1494,7 @@ define <8 x i16> @cmlez8xi16(<8 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.8h, v1.8h, v0.8h ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <8 x i16> %A, zeroinitializer; + %tmp3 = icmp sle <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -1398,7 +1510,7 @@ define <2 x i32> @cmlez2xi32(<2 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.2s, v1.2s, v0.2s ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <2 x i32> %A, zeroinitializer; + %tmp3 = icmp sle <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -1414,7 +1526,7 @@ define <4 x i32> @cmlez4xi32(<4 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.4s, v1.4s, v0.4s ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <4 x i32> %A, zeroinitializer; + %tmp3 = icmp sle <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -1430,7 +1542,120 @@ define <2 x i64> @cmlez2xi64(<2 x i64> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmge v0.2d, v1.2d, v0.2d ; CHECK-GI-NEXT: ret - %tmp3 = icmp sle <2 x i64> %A, zeroinitializer; + %tmp3 = icmp sle <2 x i64> %A, zeroinitializer + %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> + ret <2 x i64> %tmp4 +} + +define <8 x i8> @cmlez8xi8_alt(<8 x i8> %A) { +; CHECK-SD-LABEL: cmlez8xi8_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.8b, v0.8b, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez8xi8_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.8b, #1 +; CHECK-GI-NEXT: cmgt v0.8b, v1.8b, v0.8b +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <8 x i8> %A, + %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> + ret <8 x i8> %tmp4 +} + +define <16 x i8> @cmlez16xi8_alt(<16 x i8> %A) { +; CHECK-SD-LABEL: cmlez16xi8_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.16b, v0.16b, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez16xi8_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.16b, #1 +; CHECK-GI-NEXT: cmgt v0.16b, v1.16b, v0.16b +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <16 x i8> %A, + %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> + ret <16 x i8> %tmp4 +} + +define <4 x i16> @cmlez4xi16_alt(<4 x i16> %A) { +; CHECK-SD-LABEL: cmlez4xi16_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.4h, v0.4h, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez4xi16_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.4h, #1 +; CHECK-GI-NEXT: cmgt v0.4h, v1.4h, v0.4h +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <4 x i16> %A, + %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> + ret <4 x i16> %tmp4 +} + +define <8 x i16> @cmlez8xi16_alt(<8 x i16> %A) { +; CHECK-SD-LABEL: cmlez8xi16_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.8h, v0.8h, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez8xi16_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.8h, #1 +; CHECK-GI-NEXT: cmgt v0.8h, v1.8h, v0.8h +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <8 x i16> %A, + %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> + ret <8 x i16> %tmp4 +} + +define <2 x i32> @cmlez2xi32_alt(<2 x i32> %A) { +; CHECK-SD-LABEL: cmlez2xi32_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.2s, v0.2s, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez2xi32_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.2s, #1 +; CHECK-GI-NEXT: cmgt v0.2s, v1.2s, v0.2s +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <2 x i32> %A, + %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> + ret <2 x i32> %tmp4 +} + +define <4 x i32> @cmlez4xi32_alt(<4 x i32> %A) { +; CHECK-SD-LABEL: cmlez4xi32_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.4s, v0.4s, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez4xi32_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: movi v1.4s, #1 +; CHECK-GI-NEXT: cmgt v0.4s, v1.4s, v0.4s +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <4 x i32> %A, + %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> + ret <4 x i32> %tmp4 +} + +define <2 x i64> @cmlez2xi64_alt(<2 x i64> %A) { +; CHECK-SD-LABEL: cmlez2xi64_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmle v0.2d, v0.2d, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmlez2xi64_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: adrp x8, .LCPI125_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI125_0] +; CHECK-GI-NEXT: cmgt v0.2d, v1.2d, v0.2d +; CHECK-GI-NEXT: ret + %tmp3 = icmp slt <2 x i64> %A, %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -1446,7 +1671,7 @@ define <8 x i8> @cmltz8xi8(<8 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.8b, v1.8b, v0.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <8 x i8> %A, zeroinitializer; + %tmp3 = icmp slt <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -1462,7 +1687,7 @@ define <16 x i8> @cmltz16xi8(<16 x i8> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.16b, v1.16b, v0.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <16 x i8> %A, zeroinitializer; + %tmp3 = icmp slt <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -1478,7 +1703,7 @@ define <4 x i16> @cmltz4xi16(<4 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.4h, v1.4h, v0.4h ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <4 x i16> %A, zeroinitializer; + %tmp3 = icmp slt <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -1494,7 +1719,7 @@ define <8 x i16> @cmltz8xi16(<8 x i16> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.8h, v1.8h, v0.8h ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <8 x i16> %A, zeroinitializer; + %tmp3 = icmp slt <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -1510,7 +1735,7 @@ define <2 x i32> @cmltz2xi32(<2 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.2s, v1.2s, v0.2s ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <2 x i32> %A, zeroinitializer; + %tmp3 = icmp slt <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -1526,7 +1751,7 @@ define <4 x i32> @cmltz4xi32(<4 x i32> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.4s, v1.4s, v0.4s ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <4 x i32> %A, zeroinitializer; + %tmp3 = icmp slt <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -1542,11 +1767,109 @@ define <2 x i64> @cmltz2xi64(<2 x i64> %A) { ; CHECK-GI-NEXT: movi v1.2d, #0000000000000000 ; CHECK-GI-NEXT: cmgt v0.2d, v1.2d, v0.2d ; CHECK-GI-NEXT: ret - %tmp3 = icmp slt <2 x i64> %A, zeroinitializer; + %tmp3 = icmp slt <2 x i64> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } +define <8 x i8> @cmltz8xi8_alt(<8 x i8> %A) { +; CHECK-SD-LABEL: cmltz8xi8_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.8b, v0.8b, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz8xi8_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.8b, v0.8b, #7 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <8 x i8> %A, + ret <8 x i8> %A.lobit +} + +define <16 x i8> @cmltz16xi8_alt(<16 x i8> %A) { +; CHECK-SD-LABEL: cmltz16xi8_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.16b, v0.16b, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz16xi8_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.16b, v0.16b, #7 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <16 x i8> %A, + ret <16 x i8> %A.lobit +} + +define <4 x i16> @cmltz4xi16_alt(<4 x i16> %A) { +; CHECK-SD-LABEL: cmltz4xi16_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.4h, v0.4h, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz4xi16_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.4h, v0.4h, #15 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <4 x i16> %A, + ret <4 x i16> %A.lobit +} + +define <8 x i16> @cmltz8xi16_alt(<8 x i16> %A) { +; CHECK-SD-LABEL: cmltz8xi16_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.8h, v0.8h, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz8xi16_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.8h, v0.8h, #15 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <8 x i16> %A, + ret <8 x i16> %A.lobit +} + +define <2 x i32> @cmltz2xi32_alt(<2 x i32> %A) { +; CHECK-SD-LABEL: cmltz2xi32_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.2s, v0.2s, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz2xi32_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.2s, v0.2s, #31 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <2 x i32> %A, + ret <2 x i32> %A.lobit +} + +define <4 x i32> @cmltz4xi32_alt(<4 x i32> %A) { +; CHECK-SD-LABEL: cmltz4xi32_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.4s, v0.4s, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz4xi32_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.4s, v0.4s, #31 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <4 x i32> %A, + ret <4 x i32> %A.lobit +} + +define <2 x i64> @cmltz2xi64_alt(<2 x i64> %A) { +; CHECK-SD-LABEL: cmltz2xi64_alt: +; CHECK-SD: // %bb.0: +; CHECK-SD-NEXT: cmlt v0.2d, v0.2d, #0 +; CHECK-SD-NEXT: ret +; +; CHECK-GI-LABEL: cmltz2xi64_alt: +; CHECK-GI: // %bb.0: +; CHECK-GI-NEXT: sshr v0.2d, v0.2d, #63 +; CHECK-GI-NEXT: ret + %A.lobit = ashr <2 x i64> %A, + ret <2 x i64> %A.lobit +} + define <8 x i8> @cmneqz8xi8(<8 x i8> %A) { ; CHECK-SD-LABEL: cmneqz8xi8: ; CHECK-SD: // %bb.0: @@ -1559,7 +1882,7 @@ define <8 x i8> @cmneqz8xi8(<8 x i8> %A) { ; CHECK-GI-NEXT: cmeq v0.8b, v0.8b, v1.8b ; CHECK-GI-NEXT: mvn v0.8b, v0.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <8 x i8> %A, zeroinitializer; + %tmp3 = icmp ne <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -1576,7 +1899,7 @@ define <16 x i8> @cmneqz16xi8(<16 x i8> %A) { ; CHECK-GI-NEXT: cmeq v0.16b, v0.16b, v1.16b ; CHECK-GI-NEXT: mvn v0.16b, v0.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <16 x i8> %A, zeroinitializer; + %tmp3 = icmp ne <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -1593,7 +1916,7 @@ define <4 x i16> @cmneqz4xi16(<4 x i16> %A) { ; CHECK-GI-NEXT: cmeq v0.4h, v0.4h, v1.4h ; CHECK-GI-NEXT: mvn v0.8b, v0.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <4 x i16> %A, zeroinitializer; + %tmp3 = icmp ne <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -1610,7 +1933,7 @@ define <8 x i16> @cmneqz8xi16(<8 x i16> %A) { ; CHECK-GI-NEXT: cmeq v0.8h, v0.8h, v1.8h ; CHECK-GI-NEXT: mvn v0.16b, v0.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <8 x i16> %A, zeroinitializer; + %tmp3 = icmp ne <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -1627,7 +1950,7 @@ define <2 x i32> @cmneqz2xi32(<2 x i32> %A) { ; CHECK-GI-NEXT: cmeq v0.2s, v0.2s, v1.2s ; CHECK-GI-NEXT: mvn v0.8b, v0.8b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <2 x i32> %A, zeroinitializer; + %tmp3 = icmp ne <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -1644,7 +1967,7 @@ define <4 x i32> @cmneqz4xi32(<4 x i32> %A) { ; CHECK-GI-NEXT: cmeq v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: mvn v0.16b, v0.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <4 x i32> %A, zeroinitializer; + %tmp3 = icmp ne <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -1661,7 +1984,7 @@ define <2 x i64> @cmneqz2xi64(<2 x i64> %A) { ; CHECK-GI-NEXT: cmeq v0.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: mvn v0.16b, v0.16b ; CHECK-GI-NEXT: ret - %tmp3 = icmp ne <2 x i64> %A, zeroinitializer; + %tmp3 = icmp ne <2 x i64> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -1742,16 +2065,10 @@ define <2 x i64> @cmhsz2xi64(<2 x i64> %A) { ; ; CHECK-GI-LABEL: cmhsz2xi64: ; CHECK-GI: // %bb.0: -; CHECK-GI-NEXT: adrp x8, .LCPI132_0 -; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI132_0] +; CHECK-GI-NEXT: adrp x8, .LCPI153_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI153_0] ; CHECK-GI-NEXT: cmhs v0.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: ret -; GISEL-LABEL: cmhsz2xi64: -; GISEL: // %bb.0: -; GISEL-NEXT: adrp x8, .LCPI132_0 -; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI132_0] -; GISEL-NEXT: cmhs v0.2d, v0.2d, v1.2d -; GISEL-NEXT: ret %tmp3 = icmp uge <2 x i64> %A, %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 @@ -1834,16 +2151,10 @@ define <2 x i64> @cmhiz2xi64(<2 x i64> %A) { ; ; CHECK-GI-LABEL: cmhiz2xi64: ; CHECK-GI: // %bb.0: -; CHECK-GI-NEXT: adrp x8, .LCPI139_0 -; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI139_0] +; CHECK-GI-NEXT: adrp x8, .LCPI160_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI160_0] ; CHECK-GI-NEXT: cmhi v0.2d, v0.2d, v1.2d ; CHECK-GI-NEXT: ret -; GISEL-LABEL: cmhiz2xi64: -; GISEL: // %bb.0: -; GISEL-NEXT: adrp x8, .LCPI139_0 -; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI139_0] -; GISEL-NEXT: cmhi v0.2d, v0.2d, v1.2d -; GISEL-NEXT: ret %tmp3 = icmp ugt <2 x i64> %A, %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 @@ -1856,7 +2167,7 @@ define <8 x i8> @cmlsz8xi8(<8 x i8> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.8b, v1.8b, v0.8b ; CHECK-NEXT: ret - %tmp3 = icmp ule <8 x i8> %A, zeroinitializer; + %tmp3 = icmp ule <8 x i8> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i8> ret <8 x i8> %tmp4 } @@ -1868,7 +2179,7 @@ define <16 x i8> @cmlsz16xi8(<16 x i8> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.16b, v1.16b, v0.16b ; CHECK-NEXT: ret - %tmp3 = icmp ule <16 x i8> %A, zeroinitializer; + %tmp3 = icmp ule <16 x i8> %A, zeroinitializer %tmp4 = sext <16 x i1> %tmp3 to <16 x i8> ret <16 x i8> %tmp4 } @@ -1880,7 +2191,7 @@ define <4 x i16> @cmlsz4xi16(<4 x i16> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.4h, v1.4h, v0.4h ; CHECK-NEXT: ret - %tmp3 = icmp ule <4 x i16> %A, zeroinitializer; + %tmp3 = icmp ule <4 x i16> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i16> ret <4 x i16> %tmp4 } @@ -1892,7 +2203,7 @@ define <8 x i16> @cmlsz8xi16(<8 x i16> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.8h, v1.8h, v0.8h ; CHECK-NEXT: ret - %tmp3 = icmp ule <8 x i16> %A, zeroinitializer; + %tmp3 = icmp ule <8 x i16> %A, zeroinitializer %tmp4 = sext <8 x i1> %tmp3 to <8 x i16> ret <8 x i16> %tmp4 } @@ -1904,7 +2215,7 @@ define <2 x i32> @cmlsz2xi32(<2 x i32> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.2s, v1.2s, v0.2s ; CHECK-NEXT: ret - %tmp3 = icmp ule <2 x i32> %A, zeroinitializer; + %tmp3 = icmp ule <2 x i32> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i32> ret <2 x i32> %tmp4 } @@ -1916,7 +2227,7 @@ define <4 x i32> @cmlsz4xi32(<4 x i32> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret - %tmp3 = icmp ule <4 x i32> %A, zeroinitializer; + %tmp3 = icmp ule <4 x i32> %A, zeroinitializer %tmp4 = sext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 } @@ -1928,7 +2239,7 @@ define <2 x i64> @cmlsz2xi64(<2 x i64> %A) { ; CHECK-NEXT: movi v1.2d, #0000000000000000 ; CHECK-NEXT: cmhs v0.2d, v1.2d, v0.2d ; CHECK-NEXT: ret - %tmp3 = icmp ule <2 x i64> %A, zeroinitializer; + %tmp3 = icmp ule <2 x i64> %A, zeroinitializer %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 } @@ -2016,16 +2327,10 @@ define <2 x i64> @cmloz2xi64(<2 x i64> %A) { ; ; CHECK-GI-LABEL: cmloz2xi64: ; CHECK-GI: // %bb.0: -; CHECK-GI-NEXT: adrp x8, .LCPI153_0 -; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI153_0] +; CHECK-GI-NEXT: adrp x8, .LCPI174_0 +; CHECK-GI-NEXT: ldr q1, [x8, :lo12:.LCPI174_0] ; CHECK-GI-NEXT: cmhi v0.2d, v1.2d, v0.2d ; CHECK-GI-NEXT: ret -; GISEL-LABEL: cmloz2xi64: -; GISEL: // %bb.0: -; GISEL-NEXT: adrp x8, .LCPI153_0 -; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI153_0] -; GISEL-NEXT: cmhi v0.2d, v1.2d, v0.2d -; GISEL-NEXT: ret %tmp3 = icmp ult <2 x i64> %A, %tmp4 = sext <2 x i1> %tmp3 to <2 x i64> ret <2 x i64> %tmp4 @@ -4175,13 +4480,6 @@ define <4 x i32> @fcmule4xfloat_fast_zext(<4 x float> %A, <4 x float> %B) { ; CHECK-GI-NEXT: fcmgt v0.4s, v0.4s, v1.4s ; CHECK-GI-NEXT: bic v0.16b, v2.16b, v0.16b ; CHECK-GI-NEXT: ret -; GISEL-LABEL: fcmule4xfloat_fast_zext: -; GISEL: // %bb.0: -; GISEL-NEXT: fcmgt v0.4s, v0.4s, v1.4s -; GISEL-NEXT: adrp x8, .LCPI322_0 -; GISEL-NEXT: ldr q1, [x8, :lo12:.LCPI322_0] -; GISEL-NEXT: bic v0.16b, v1.16b, v0.16b -; GISEL-NEXT: ret %tmp3 = fcmp fast ule <4 x float> %A, %B %tmp4 = zext <4 x i1> %tmp3 to <4 x i32> ret <4 x i32> %tmp4 diff --git a/llvm/test/CodeGen/AArch64/signbit-shift.ll b/llvm/test/CodeGen/AArch64/signbit-shift.ll index cb758f8a6202..253ea1cab91f 100644 --- a/llvm/test/CodeGen/AArch64/signbit-shift.ll +++ b/llvm/test/CodeGen/AArch64/signbit-shift.ll @@ -29,10 +29,9 @@ define i32 @add_zext_ifpos(i32 %x) { define <4 x i32> @add_zext_ifpos_vec_splat(<4 x i32> %x) { ; CHECK-LABEL: add_zext_ifpos_vec_splat: ; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0xffffffffffffffff -; CHECK-NEXT: movi v2.4s, #41 -; CHECK-NEXT: cmgt v0.4s, v0.4s, v1.4s -; CHECK-NEXT: sub v0.4s, v2.4s, v0.4s +; CHECK-NEXT: movi v1.4s, #41 +; CHECK-NEXT: cmge v0.4s, v0.4s, #0 +; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s ; CHECK-NEXT: ret %c = icmp sgt <4 x i32> %x, %e = zext <4 x i1> %c to <4 x i32> @@ -43,7 +42,7 @@ define <4 x i32> @add_zext_ifpos_vec_splat(<4 x i32> %x) { define i32 @sel_ifpos_tval_bigger(i32 %x) { ; CHECK-LABEL: sel_ifpos_tval_bigger: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #41 +; CHECK-NEXT: mov w8, #41 // =0x29 ; CHECK-NEXT: cmp w0, #0 ; CHECK-NEXT: cinc w0, w8, ge ; CHECK-NEXT: ret @@ -78,10 +77,9 @@ define i32 @add_sext_ifpos(i32 %x) { define <4 x i32> @add_sext_ifpos_vec_splat(<4 x i32> %x) { ; CHECK-LABEL: add_sext_ifpos_vec_splat: ; CHECK: // %bb.0: -; CHECK-NEXT: movi v1.2d, #0xffffffffffffffff -; CHECK-NEXT: movi v2.4s, #42 -; CHECK-NEXT: cmgt v0.4s, v0.4s, v1.4s -; CHECK-NEXT: add v0.4s, v0.4s, v2.4s +; CHECK-NEXT: movi v1.4s, #42 +; CHECK-NEXT: cmge v0.4s, v0.4s, #0 +; CHECK-NEXT: add v0.4s, v0.4s, v1.4s ; CHECK-NEXT: ret %c = icmp sgt <4 x i32> %x, %e = sext <4 x i1> %c to <4 x i32> @@ -92,7 +90,7 @@ define <4 x i32> @add_sext_ifpos_vec_splat(<4 x i32> %x) { define i32 @sel_ifpos_fval_bigger(i32 %x) { ; CHECK-LABEL: sel_ifpos_fval_bigger: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #41 +; CHECK-NEXT: mov w8, #41 // =0x29 ; CHECK-NEXT: cmp w0, #0 ; CHECK-NEXT: cinc w0, w8, lt ; CHECK-NEXT: ret @@ -128,7 +126,7 @@ define i32 @add_zext_ifneg(i32 %x) { define i32 @sel_ifneg_tval_bigger(i32 %x) { ; CHECK-LABEL: sel_ifneg_tval_bigger: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #41 +; CHECK-NEXT: mov w8, #41 // =0x29 ; CHECK-NEXT: cmp w0, #0 ; CHECK-NEXT: cinc w0, w8, lt ; CHECK-NEXT: ret @@ -162,7 +160,7 @@ define i32 @add_sext_ifneg(i32 %x) { define i32 @sel_ifneg_fval_bigger(i32 %x) { ; CHECK-LABEL: sel_ifneg_fval_bigger: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #41 +; CHECK-NEXT: mov w8, #41 // =0x29 ; CHECK-NEXT: cmp w0, #0 ; CHECK-NEXT: cinc w0, w8, ge ; CHECK-NEXT: ret @@ -199,7 +197,7 @@ define <4 x i32> @add_lshr_not_vec_splat(<4 x i32> %x) { define i32 @sub_lshr_not(i32 %x) { ; CHECK-LABEL: sub_lshr_not: ; CHECK: // %bb.0: -; CHECK-NEXT: mov w8, #42 +; CHECK-NEXT: mov w8, #42 // =0x2a ; CHECK-NEXT: bfxil w8, w0, #31, #1 ; CHECK-NEXT: mov w0, w8 ; CHECK-NEXT: ret diff --git a/llvm/test/CodeGen/AArch64/stack-probing-64k.ll b/llvm/test/CodeGen/AArch64/stack-probing-64k.ll index 945c271d3750..2f15e317a7f5 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-64k.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing-64k.ll @@ -313,7 +313,7 @@ define void @static_16_align_131072(ptr %out) #0 { ; CHECK-NEXT: b .LBB9_1 ; CHECK-NEXT: .LBB9_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: mov x8, sp ; CHECK-NEXT: str x8, [x0] ; CHECK-NEXT: mov sp, x29 diff --git a/llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll b/llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll index d247ed1b5997..a3b8df487ed4 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing-dynamic.ll @@ -28,7 +28,7 @@ define void @dynamic(i64 %size, ptr %out) #0 { ; CHECK-NEXT: b .LBB0_1 ; CHECK-NEXT: .LBB0_3: ; CHECK-NEXT: mov sp, x8 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 @@ -72,7 +72,7 @@ define void @dynamic_fixed(i64 %size, ptr %out1, ptr %out2) #0 { ; CHECK-NEXT: b .LBB1_1 ; CHECK-NEXT: .LBB1_3: ; CHECK-NEXT: mov sp, x8 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: str x8, [x2] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 @@ -122,7 +122,7 @@ define void @dynamic_align_64(i64 %size, ptr %out) #0 { ; CHECK-NEXT: b .LBB2_1 ; CHECK-NEXT: .LBB2_3: ; CHECK-NEXT: mov sp, x8 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 32 @@ -167,7 +167,7 @@ define void @dynamic_align_8192(i64 %size, ptr %out) #0 { ; CHECK-NEXT: mov sp, x9 ; CHECK-NEXT: add x9, x0, #15 ; CHECK-NEXT: mov x8, sp -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: and x9, x9, #0xfffffffffffffff0 ; CHECK-NEXT: mov x19, sp ; CHECK-NEXT: sub x8, x8, x9 @@ -181,7 +181,7 @@ define void @dynamic_align_8192(i64 %size, ptr %out) #0 { ; CHECK-NEXT: b .LBB3_4 ; CHECK-NEXT: .LBB3_6: ; CHECK-NEXT: mov sp, x8 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 32 @@ -221,7 +221,7 @@ define void @dynamic_64k_guard(i64 %size, ptr %out) #0 "stack-probe-size"="65536 ; CHECK-NEXT: b .LBB4_1 ; CHECK-NEXT: .LBB4_3: ; CHECK-NEXT: mov sp, x8 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 @@ -265,7 +265,7 @@ define void @no_reserved_call_frame(i64 %n) #0 { ; CHECK-NEXT: b .LBB5_1 ; CHECK-NEXT: .LBB5_3: // %entry ; CHECK-NEXT: mov sp, x0 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: sub sp, sp, #1104 ; CHECK-NEXT: str xzr, [sp] ; CHECK-NEXT: bl callee_stack_args @@ -344,7 +344,7 @@ define void @dynamic_sve(i64 %size, ptr %out) #0 "target-features"="+sve" { ; CHECK-NEXT: b .LBB7_1 ; CHECK-NEXT: .LBB7_3: ; CHECK-NEXT: mov sp, x8 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: str x8, [x1] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 32 diff --git a/llvm/test/CodeGen/AArch64/stack-probing-sve.ll b/llvm/test/CodeGen/AArch64/stack-probing-sve.ll index 4dad104e66f2..03a9220ebfdd 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing-sve.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing-sve.ll @@ -115,7 +115,7 @@ define void @sve_17_vector(ptr %out) #0 { ; CHECK-NEXT: b .LBB3_1 ; CHECK-NEXT: .LBB3_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: .cfi_def_cfa_register wsp ; CHECK-NEXT: addvl sp, sp, #17 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 @@ -351,7 +351,7 @@ define void @sve_16v_1p_csr( %a) #0 { ; CHECK-NEXT: b .LBB9_1 ; CHECK-NEXT: .LBB9_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: .cfi_def_cfa_register wsp ; CHECK-NEXT: str p8, [sp, #7, mul vl] // 2-byte Folded Spill ; CHECK-NEXT: str z23, [sp, #1, mul vl] // 16-byte Folded Spill @@ -467,7 +467,7 @@ define void @sve_1_vector_4096_arr(ptr %out) #0 { ; CHECK-NEXT: b .LBB11_1 ; CHECK-NEXT: .LBB11_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: .cfi_def_cfa_register wsp ; CHECK-NEXT: addvl sp, sp, #31 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0f, 0x8f, 0x00, 0x11, 0x90, 0xe0, 0x00, 0x22, 0x11, 0x88, 0x02, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 12304 + 264 * VG @@ -516,7 +516,7 @@ define void @sve_1_vector_16_arr_align_8192(ptr %out) #0 { ; CHECK-NEXT: b .LBB12_1 ; CHECK-NEXT: .LBB12_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: mov sp, x29 ; CHECK-NEXT: .cfi_def_cfa wsp, 16 ; CHECK-NEXT: ldp x29, x30, [sp], #16 // 16-byte Folded Reload @@ -616,7 +616,7 @@ define void @sve_1028_64k_guard(ptr %out) #0 "stack-probe-size"="65536" { ; CHECK-NEXT: b .LBB14_1 ; CHECK-NEXT: .LBB14_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: .cfi_def_cfa_register wsp ; CHECK-NEXT: addvl sp, sp, #31 ; CHECK-NEXT: .cfi_escape 0x0f, 0x0d, 0x8f, 0x00, 0x11, 0x10, 0x22, 0x11, 0x90, 0x0e, 0x92, 0x2e, 0x00, 0x1e, 0x22 // sp + 16 + 1808 * VG diff --git a/llvm/test/CodeGen/AArch64/stack-probing.ll b/llvm/test/CodeGen/AArch64/stack-probing.ll index 5c5d9321a56e..df5408de5bab 100644 --- a/llvm/test/CodeGen/AArch64/stack-probing.ll +++ b/llvm/test/CodeGen/AArch64/stack-probing.ll @@ -400,7 +400,7 @@ define void @static_16_align_8192(ptr %out) #0 { ; CHECK-NEXT: b .LBB13_1 ; CHECK-NEXT: .LBB13_3: // %entry ; CHECK-NEXT: mov sp, x9 -; CHECK-NEXT: str xzr, [sp] +; CHECK-NEXT: ldr xzr, [sp] ; CHECK-NEXT: mov x8, sp ; CHECK-NEXT: str x8, [x0] ; CHECK-NEXT: mov sp, x29 diff --git a/llvm/test/CodeGen/AArch64/vselect-ext.ll b/llvm/test/CodeGen/AArch64/vselect-ext.ll index 0ae09ebe9163..b80955665c74 100644 --- a/llvm/test/CodeGen/AArch64/vselect-ext.ll +++ b/llvm/test/CodeGen/AArch64/vselect-ext.ll @@ -543,15 +543,14 @@ entry: define <16 x i32> @same_zext_used_in_cmp_signed_pred_and_select_can_convert_to_unsigned_pred(<16 x i8> %a) { ; CHECK-LABEL: same_zext_used_in_cmp_signed_pred_and_select_can_convert_to_unsigned_pred: ; CHECK: ; %bb.0: ; %entry -; CHECK-NEXT: movi.2d v1, #0xffffffffffffffff +; CHECK-NEXT: cmge.16b v1, v0, #0 ; CHECK-NEXT: ushll.8h v2, v0, #0 -; CHECK-NEXT: ushll.4s v4, v2, #0 -; CHECK-NEXT: ushll2.4s v2, v2, #0 -; CHECK-NEXT: cmgt.16b v1, v0, v1 ; CHECK-NEXT: ushll2.8h v0, v0, #0 ; CHECK-NEXT: sshll.8h v3, v1, #0 ; CHECK-NEXT: sshll2.8h v1, v1, #0 +; CHECK-NEXT: ushll.4s v4, v2, #0 ; CHECK-NEXT: ushll.4s v5, v0, #0 +; CHECK-NEXT: ushll2.4s v2, v2, #0 ; CHECK-NEXT: ushll2.4s v6, v0, #0 ; CHECK-NEXT: sshll.4s v0, v3, #0 ; CHECK-NEXT: sshll.4s v7, v1, #0 @@ -574,52 +573,51 @@ define void @extension_in_loop_v16i8_to_v16i32(ptr %src, ptr %dst) { ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: Lloh2: ; CHECK-NEXT: adrp x8, lCPI24_0@PAGE -; CHECK-NEXT: movi.2d v0, #0xffffffffffffffff ; CHECK-NEXT: Lloh3: -; CHECK-NEXT: adrp x9, lCPI24_2@PAGE +; CHECK-NEXT: adrp x9, lCPI24_1@PAGE ; CHECK-NEXT: Lloh4: -; CHECK-NEXT: ldr q1, [x8, lCPI24_0@PAGEOFF] +; CHECK-NEXT: adrp x10, lCPI24_2@PAGE ; CHECK-NEXT: Lloh5: -; CHECK-NEXT: adrp x8, lCPI24_1@PAGE +; CHECK-NEXT: ldr q0, [x8, lCPI24_0@PAGEOFF] ; CHECK-NEXT: Lloh6: -; CHECK-NEXT: adrp x10, lCPI24_3@PAGE +; CHECK-NEXT: adrp x8, lCPI24_3@PAGE ; CHECK-NEXT: Lloh7: -; CHECK-NEXT: ldr q2, [x8, lCPI24_1@PAGEOFF] +; CHECK-NEXT: ldr q1, [x9, lCPI24_1@PAGEOFF] ; CHECK-NEXT: Lloh8: -; CHECK-NEXT: ldr q3, [x9, lCPI24_2@PAGEOFF] +; CHECK-NEXT: ldr q2, [x10, lCPI24_2@PAGEOFF] ; CHECK-NEXT: Lloh9: -; CHECK-NEXT: ldr q4, [x10, lCPI24_3@PAGEOFF] +; CHECK-NEXT: ldr q3, [x8, lCPI24_3@PAGEOFF] ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: LBB24_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ldr q5, [x0, x8] +; CHECK-NEXT: ldr q4, [x0, x8] ; CHECK-NEXT: add x8, x8, #16 ; CHECK-NEXT: cmp x8, #128 -; CHECK-NEXT: cmgt.16b v6, v5, v0 -; CHECK-NEXT: tbl.16b v16, { v5 }, v1 -; CHECK-NEXT: tbl.16b v17, { v5 }, v2 -; CHECK-NEXT: tbl.16b v19, { v5 }, v3 -; CHECK-NEXT: tbl.16b v5, { v5 }, v4 -; CHECK-NEXT: sshll2.8h v7, v6, #0 -; CHECK-NEXT: sshll.8h v6, v6, #0 -; CHECK-NEXT: sshll2.4s v18, v7, #0 -; CHECK-NEXT: sshll.4s v7, v7, #0 -; CHECK-NEXT: sshll2.4s v20, v6, #0 +; CHECK-NEXT: cmge.16b v5, v4, #0 +; CHECK-NEXT: tbl.16b v7, { v4 }, v0 +; CHECK-NEXT: tbl.16b v16, { v4 }, v1 +; CHECK-NEXT: tbl.16b v18, { v4 }, v2 +; CHECK-NEXT: tbl.16b v4, { v4 }, v3 +; CHECK-NEXT: sshll2.8h v6, v5, #0 +; CHECK-NEXT: sshll.8h v5, v5, #0 +; CHECK-NEXT: sshll2.4s v17, v6, #0 ; CHECK-NEXT: sshll.4s v6, v6, #0 -; CHECK-NEXT: and.16b v16, v16, v18 -; CHECK-NEXT: and.16b v7, v17, v7 -; CHECK-NEXT: and.16b v17, v19, v20 -; CHECK-NEXT: and.16b v5, v5, v6 -; CHECK-NEXT: stp q7, q16, [x1, #32] -; CHECK-NEXT: stp q5, q17, [x1], #64 +; CHECK-NEXT: sshll2.4s v19, v5, #0 +; CHECK-NEXT: sshll.4s v5, v5, #0 +; CHECK-NEXT: and.16b v7, v7, v17 +; CHECK-NEXT: and.16b v6, v16, v6 +; CHECK-NEXT: and.16b v16, v18, v19 +; CHECK-NEXT: and.16b v4, v4, v5 +; CHECK-NEXT: stp q6, q7, [x1, #32] +; CHECK-NEXT: stp q4, q16, [x1], #64 ; CHECK-NEXT: b.ne LBB24_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret ; CHECK-NEXT: .loh AdrpLdr Lloh6, Lloh9 -; CHECK-NEXT: .loh AdrpLdr Lloh5, Lloh7 -; CHECK-NEXT: .loh AdrpLdr Lloh3, Lloh8 -; CHECK-NEXT: .loh AdrpAdrp Lloh2, Lloh5 -; CHECK-NEXT: .loh AdrpLdr Lloh2, Lloh4 +; CHECK-NEXT: .loh AdrpLdr Lloh4, Lloh8 +; CHECK-NEXT: .loh AdrpLdr Lloh3, Lloh7 +; CHECK-NEXT: .loh AdrpAdrp Lloh2, Lloh6 +; CHECK-NEXT: .loh AdrpLdr Lloh2, Lloh5 entry: br label %loop @@ -645,52 +643,51 @@ define void @extension_in_loop_as_shuffle_v16i8_to_v16i32(ptr %src, ptr %dst) { ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: Lloh10: ; CHECK-NEXT: adrp x8, lCPI25_0@PAGE -; CHECK-NEXT: movi.2d v0, #0xffffffffffffffff ; CHECK-NEXT: Lloh11: -; CHECK-NEXT: adrp x9, lCPI25_2@PAGE +; CHECK-NEXT: adrp x9, lCPI25_1@PAGE ; CHECK-NEXT: Lloh12: -; CHECK-NEXT: ldr q1, [x8, lCPI25_0@PAGEOFF] +; CHECK-NEXT: adrp x10, lCPI25_2@PAGE ; CHECK-NEXT: Lloh13: -; CHECK-NEXT: adrp x8, lCPI25_1@PAGE +; CHECK-NEXT: ldr q0, [x8, lCPI25_0@PAGEOFF] ; CHECK-NEXT: Lloh14: -; CHECK-NEXT: adrp x10, lCPI25_3@PAGE +; CHECK-NEXT: adrp x8, lCPI25_3@PAGE ; CHECK-NEXT: Lloh15: -; CHECK-NEXT: ldr q2, [x8, lCPI25_1@PAGEOFF] +; CHECK-NEXT: ldr q1, [x9, lCPI25_1@PAGEOFF] ; CHECK-NEXT: Lloh16: -; CHECK-NEXT: ldr q3, [x9, lCPI25_2@PAGEOFF] +; CHECK-NEXT: ldr q2, [x10, lCPI25_2@PAGEOFF] ; CHECK-NEXT: Lloh17: -; CHECK-NEXT: ldr q4, [x10, lCPI25_3@PAGEOFF] +; CHECK-NEXT: ldr q3, [x8, lCPI25_3@PAGEOFF] ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: LBB25_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ldr q5, [x0, x8] +; CHECK-NEXT: ldr q4, [x0, x8] ; CHECK-NEXT: add x8, x8, #16 ; CHECK-NEXT: cmp x8, #128 -; CHECK-NEXT: cmgt.16b v6, v5, v0 -; CHECK-NEXT: tbl.16b v16, { v5 }, v1 -; CHECK-NEXT: tbl.16b v17, { v5 }, v2 -; CHECK-NEXT: tbl.16b v19, { v5 }, v3 -; CHECK-NEXT: tbl.16b v5, { v5 }, v4 -; CHECK-NEXT: sshll2.8h v7, v6, #0 -; CHECK-NEXT: sshll.8h v6, v6, #0 -; CHECK-NEXT: sshll2.4s v18, v7, #0 -; CHECK-NEXT: sshll.4s v7, v7, #0 -; CHECK-NEXT: sshll2.4s v20, v6, #0 +; CHECK-NEXT: cmge.16b v5, v4, #0 +; CHECK-NEXT: tbl.16b v7, { v4 }, v0 +; CHECK-NEXT: tbl.16b v16, { v4 }, v1 +; CHECK-NEXT: tbl.16b v18, { v4 }, v2 +; CHECK-NEXT: tbl.16b v4, { v4 }, v3 +; CHECK-NEXT: sshll2.8h v6, v5, #0 +; CHECK-NEXT: sshll.8h v5, v5, #0 +; CHECK-NEXT: sshll2.4s v17, v6, #0 ; CHECK-NEXT: sshll.4s v6, v6, #0 -; CHECK-NEXT: and.16b v16, v16, v18 -; CHECK-NEXT: and.16b v7, v17, v7 -; CHECK-NEXT: and.16b v17, v19, v20 -; CHECK-NEXT: and.16b v5, v5, v6 -; CHECK-NEXT: stp q7, q16, [x1, #32] -; CHECK-NEXT: stp q5, q17, [x1], #64 +; CHECK-NEXT: sshll2.4s v19, v5, #0 +; CHECK-NEXT: sshll.4s v5, v5, #0 +; CHECK-NEXT: and.16b v7, v7, v17 +; CHECK-NEXT: and.16b v6, v16, v6 +; CHECK-NEXT: and.16b v16, v18, v19 +; CHECK-NEXT: and.16b v4, v4, v5 +; CHECK-NEXT: stp q6, q7, [x1, #32] +; CHECK-NEXT: stp q4, q16, [x1], #64 ; CHECK-NEXT: b.ne LBB25_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret ; CHECK-NEXT: .loh AdrpLdr Lloh14, Lloh17 -; CHECK-NEXT: .loh AdrpLdr Lloh13, Lloh15 -; CHECK-NEXT: .loh AdrpLdr Lloh11, Lloh16 -; CHECK-NEXT: .loh AdrpAdrp Lloh10, Lloh13 -; CHECK-NEXT: .loh AdrpLdr Lloh10, Lloh12 +; CHECK-NEXT: .loh AdrpLdr Lloh12, Lloh16 +; CHECK-NEXT: .loh AdrpLdr Lloh11, Lloh15 +; CHECK-NEXT: .loh AdrpAdrp Lloh10, Lloh14 +; CHECK-NEXT: .loh AdrpLdr Lloh10, Lloh13 entry: br label %loop @@ -717,52 +714,51 @@ define void @shuffle_in_loop_is_no_extend_v16i8_to_v16i32(ptr %src, ptr %dst) { ; CHECK: ; %bb.0: ; %entry ; CHECK-NEXT: Lloh18: ; CHECK-NEXT: adrp x8, lCPI26_0@PAGE -; CHECK-NEXT: movi.2d v0, #0xffffffffffffffff ; CHECK-NEXT: Lloh19: -; CHECK-NEXT: adrp x9, lCPI26_2@PAGE +; CHECK-NEXT: adrp x9, lCPI26_1@PAGE ; CHECK-NEXT: Lloh20: -; CHECK-NEXT: ldr q1, [x8, lCPI26_0@PAGEOFF] +; CHECK-NEXT: adrp x10, lCPI26_2@PAGE ; CHECK-NEXT: Lloh21: -; CHECK-NEXT: adrp x8, lCPI26_1@PAGE +; CHECK-NEXT: ldr q0, [x8, lCPI26_0@PAGEOFF] ; CHECK-NEXT: Lloh22: -; CHECK-NEXT: adrp x10, lCPI26_3@PAGE +; CHECK-NEXT: adrp x8, lCPI26_3@PAGE ; CHECK-NEXT: Lloh23: -; CHECK-NEXT: ldr q2, [x8, lCPI26_1@PAGEOFF] +; CHECK-NEXT: ldr q1, [x9, lCPI26_1@PAGEOFF] ; CHECK-NEXT: Lloh24: -; CHECK-NEXT: ldr q3, [x9, lCPI26_2@PAGEOFF] +; CHECK-NEXT: ldr q2, [x10, lCPI26_2@PAGEOFF] ; CHECK-NEXT: Lloh25: -; CHECK-NEXT: ldr q4, [x10, lCPI26_3@PAGEOFF] +; CHECK-NEXT: ldr q3, [x8, lCPI26_3@PAGEOFF] ; CHECK-NEXT: mov x8, xzr ; CHECK-NEXT: LBB26_1: ; %loop ; CHECK-NEXT: ; =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: ldr q5, [x0, x8] +; CHECK-NEXT: ldr q4, [x0, x8] ; CHECK-NEXT: add x8, x8, #16 ; CHECK-NEXT: cmp x8, #128 -; CHECK-NEXT: cmgt.16b v6, v5, v0 -; CHECK-NEXT: tbl.16b v16, { v5 }, v1 -; CHECK-NEXT: tbl.16b v17, { v5 }, v2 -; CHECK-NEXT: tbl.16b v19, { v5 }, v3 -; CHECK-NEXT: tbl.16b v5, { v5 }, v4 -; CHECK-NEXT: sshll2.8h v7, v6, #0 -; CHECK-NEXT: sshll.8h v6, v6, #0 -; CHECK-NEXT: sshll2.4s v18, v7, #0 -; CHECK-NEXT: sshll.4s v7, v7, #0 -; CHECK-NEXT: sshll2.4s v20, v6, #0 +; CHECK-NEXT: cmge.16b v5, v4, #0 +; CHECK-NEXT: tbl.16b v7, { v4 }, v0 +; CHECK-NEXT: tbl.16b v16, { v4 }, v1 +; CHECK-NEXT: tbl.16b v18, { v4 }, v2 +; CHECK-NEXT: tbl.16b v4, { v4 }, v3 +; CHECK-NEXT: sshll2.8h v6, v5, #0 +; CHECK-NEXT: sshll.8h v5, v5, #0 +; CHECK-NEXT: sshll2.4s v17, v6, #0 ; CHECK-NEXT: sshll.4s v6, v6, #0 -; CHECK-NEXT: and.16b v16, v16, v18 -; CHECK-NEXT: and.16b v7, v17, v7 -; CHECK-NEXT: and.16b v17, v19, v20 -; CHECK-NEXT: and.16b v5, v5, v6 -; CHECK-NEXT: stp q7, q16, [x1, #32] -; CHECK-NEXT: stp q5, q17, [x1], #64 +; CHECK-NEXT: sshll2.4s v19, v5, #0 +; CHECK-NEXT: sshll.4s v5, v5, #0 +; CHECK-NEXT: and.16b v7, v7, v17 +; CHECK-NEXT: and.16b v6, v16, v6 +; CHECK-NEXT: and.16b v16, v18, v19 +; CHECK-NEXT: and.16b v4, v4, v5 +; CHECK-NEXT: stp q6, q7, [x1, #32] +; CHECK-NEXT: stp q4, q16, [x1], #64 ; CHECK-NEXT: b.ne LBB26_1 ; CHECK-NEXT: ; %bb.2: ; %exit ; CHECK-NEXT: ret ; CHECK-NEXT: .loh AdrpLdr Lloh22, Lloh25 -; CHECK-NEXT: .loh AdrpLdr Lloh21, Lloh23 -; CHECK-NEXT: .loh AdrpLdr Lloh19, Lloh24 -; CHECK-NEXT: .loh AdrpAdrp Lloh18, Lloh21 -; CHECK-NEXT: .loh AdrpLdr Lloh18, Lloh20 +; CHECK-NEXT: .loh AdrpLdr Lloh20, Lloh24 +; CHECK-NEXT: .loh AdrpLdr Lloh19, Lloh23 +; CHECK-NEXT: .loh AdrpAdrp Lloh18, Lloh22 +; CHECK-NEXT: .loh AdrpLdr Lloh18, Lloh21 entry: br label %loop diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-prefetch.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-prefetch.ll new file mode 100644 index 000000000000..b53610a0f22e --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/irtranslator-prefetch.ll @@ -0,0 +1,32 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -global-isel -mtriple=amdgcn -verify-machineinstrs -stop-after=irtranslator < %s | FileCheck %s + +define void @prefetch_read(ptr %ptr) { + ; CHECK-LABEL: name: prefetch_read + ; CHECK: bb.1 (%ir-block.0): + ; CHECK-NEXT: liveins: $vgpr0, $vgpr1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 + ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32) + ; CHECK-NEXT: G_PREFETCH [[MV]](p0), 0, 0, 0 :: (load unknown-size from %ir.ptr, align 1) + ; CHECK-NEXT: SI_RETURN + call void @llvm.prefetch.p0(ptr %ptr, i32 0, i32 0, i32 0) + ret void +} + +define void @prefetch_write(ptr %ptr) { + ; CHECK-LABEL: name: prefetch_write + ; CHECK: bb.1 (%ir-block.0): + ; CHECK-NEXT: liveins: $vgpr0, $vgpr1 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $vgpr0 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $vgpr1 + ; CHECK-NEXT: [[MV:%[0-9]+]]:_(p0) = G_MERGE_VALUES [[COPY]](s32), [[COPY1]](s32) + ; CHECK-NEXT: G_PREFETCH [[MV]](p0), 1, 1, 1 :: (store unknown-size into %ir.ptr, align 1) + ; CHECK-NEXT: SI_RETURN + call void @llvm.prefetch.p0(ptr %ptr, i32 1, i32 1, i32 1) + ret void +} + +declare void @llvm.prefetch.p0(ptr, i32, i32, i32) diff --git a/llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll b/llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll index 4760ddb65bf2..680fae186960 100644 --- a/llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll +++ b/llvm/test/CodeGen/AMDGPU/codegen-internal-only-func.ll @@ -1,6 +1,7 @@ ; REQUIRES: asserts -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck -check-prefix=OPT %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck -check-prefixes=OPT,COV4 %s ; RUN: not llc --crash -O0 -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -filetype=null %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck -check-prefixes=OPT,COV5 %s ; AMDGPUAttributor deletes the function "by accident" so it's never ; codegened with optimizations. @@ -14,9 +15,12 @@ ; OPT-NEXT: amdhsa.target: amdgcn-amd-amdhsa--gfx900 ; OPT-NEXT: amdhsa.version: ; OPT-NEXT: - 1 -; OPT-NEXT: - 1 -; OPT-NEXT: ... +; COV4: - 1 +; COV5: - 2 +; OPT: ... define internal i32 @func() { ret i32 0 } +!llvm.module.flags = !{!0} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll b/llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll index e5de7859dab6..f8fc3e1e7648 100644 --- a/llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll +++ b/llvm/test/CodeGen/AMDGPU/elf-header-osabi.ll @@ -1,9 +1,12 @@ ; RUN: llc -filetype=obj -mtriple=amdgcn -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=NONE %s ; RUN: llc -filetype=obj -mtriple=amdgcn-amd- -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=NONE %s ; RUN: llc -filetype=obj -mtriple=amdgcn-amd-unknown -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=NONE %s -; RUN: llc -filetype=obj -mtriple=amdgcn--amdhsa -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA %s -; RUN: llc -filetype=obj -mtriple=amdgcn-amd-amdhsa -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA %s -; RUN: llc -filetype=obj -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA %s +; RUN: llc -filetype=obj -mtriple=amdgcn--amdhsa -mcpu=gfx801 --amdhsa-code-object-version=4 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA,HSA4 %s +; RUN: llc -filetype=obj -mtriple=amdgcn-amd-amdhsa -mcpu=gfx801 --amdhsa-code-object-version=4 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA,HSA4 %s +; RUN: llc -filetype=obj -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx801 --amdhsa-code-object-version=4 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA,HSA4 %s +; RUN: llc -filetype=obj -mtriple=amdgcn--amdhsa -mcpu=gfx801 --amdhsa-code-object-version=5 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA,HSA5 %s +; RUN: llc -filetype=obj -mtriple=amdgcn-amd-amdhsa -mcpu=gfx801 --amdhsa-code-object-version=5 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA,HSA5 %s +; RUN: llc -filetype=obj -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx801 --amdhsa-code-object-version=5 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=HSA,HSA5 %s ; RUN: llc -filetype=obj -mtriple=amdgcn--amdpal -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=PAL %s ; RUN: llc -filetype=obj -mtriple=amdgcn-amd-amdpal -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=PAL %s ; RUN: llc -filetype=obj -mtriple=amdgcn-unknown-amdpal -mcpu=gfx801 < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=PAL %s @@ -13,7 +16,8 @@ ; NONE: OS/ABI: SystemV (0x0) ; HSA: OS/ABI: AMDGPU_HSA (0x40) -; HSA: ABIVersion: 2 +; HSA4: ABIVersion: 2 +; HSA5: ABIVersion: 3 ; PAL: OS/ABI: AMDGPU_PAL (0x41) ; PAL: ABIVersion: 0 ; MESA3D: OS/ABI: AMDGPU_MESA3D (0x42) diff --git a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll index 61017e809c86..e2a3749c7c47 100644 --- a/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll +++ b/llvm/test/CodeGen/AMDGPU/integer-mad-patterns.ll @@ -8,8 +8,8 @@ ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx803 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s ; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx803 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s -; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX900,GFX9-SDAG,GFX900-SDAG %s -; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX900,GFX9-GISEL,GFX900-GISEL %s +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG,GFX900-SDAG,GFX900 %s +; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx900 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL,GFX900-GISEL,GFX900 %s ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX9,GFX90A,GFX9-SDAG,GFX90A-SDAG %s ; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx90a < %s | FileCheck -check-prefixes=GFX9,GFX90A,GFX9-GISEL,GFX90A-GISEL %s @@ -17,6 +17,9 @@ ; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s ; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1030 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s +; RUN: llc -global-isel=0 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s +; RUN: llc -global-isel=1 -mtriple=amdgcn-amd-amdpal -mcpu=gfx1100 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s + ; Test for integer mad formation for patterns used in clpeak define i32 @clpeak_imad_pat_i32(i32 %x, i32 %y) { @@ -143,6 +146,36 @@ define i32 @clpeak_imad_pat_i32(i32 %x, i32 %y) { ; GFX10-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i32: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v0, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v5, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v2, v5, v[2:3] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v3, v2, v[3:4] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i32: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add i32 %x, 1 %add = mul i32 %y18, %y @@ -271,6 +304,35 @@ define signext i16 @clpeak_imad_pat_i16(i16 signext %x, i16 signext %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 16 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v0 +; GFX11-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v2, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v2, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %conv33 = add i16 %x, 1 %add = mul i16 %conv33, %y @@ -462,6 +524,38 @@ define <2 x i16> @clpeak_imad_pat_v2i16(<2 x i16> %x, <2 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v2, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <2 x i16> %x, %add = mul <2 x i16> %y18, %y @@ -719,6 +813,60 @@ define <3 x i16> @clpeak_imad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v3i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v1, -1 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v4, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v5, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v5, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v3, v4, -1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v4, v1, -1 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v5, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v3i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v4, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y48 = add <3 x i16> %x, %add = mul <3 x i16> %y48, %y @@ -1062,6 +1210,60 @@ define <4 x i16> @clpeak_imad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v4i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v1, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v4, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v5, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v5, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v3, v4, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v4, v1, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v5, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v4i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v4, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <4 x i16> %x, %add = mul <4 x i16> %y18, %y @@ -1184,6 +1386,35 @@ define zeroext i16 @clpeak_umad_pat_i16(i16 zeroext %x, i16 zeroext %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_umad_pat_i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v0 +; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_umad_pat_i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v2, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v2, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %conv33 = add i16 %x, 1 %add = mul i16 %conv33, %y @@ -1375,6 +1606,38 @@ define <2 x i16> @clpeak_umad_pat_v2i16(<2 x i16> %x, <2 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_umad_pat_v2i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v2, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_umad_pat_v2i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <2 x i16> %x, %add = mul <2 x i16> %y18, %y @@ -1632,6 +1895,60 @@ define <3 x i16> @clpeak_umad_pat_v3i16(<3 x i16> %x, <3 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_umad_pat_v3i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v1, -1 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v4, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v5, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v5, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v3, v4, -1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v4, v1, -1 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v5, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_umad_pat_v3i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v4, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y48 = add <3 x i16> %x, %add = mul <3 x i16> %y48, %y @@ -1975,6 +2292,60 @@ define <4 x i16> @clpeak_umad_pat_v4i16(<4 x i16> %x, <4 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_umad_pat_v4i16: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v1, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v4, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v5, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v4, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v5, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v5, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v3, v4, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v4, v1, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v5, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v5 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v4 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_umad_pat_v4i16: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v4, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v4, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v4, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v3, v5, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_add_u16 v4, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v5, v1, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <4 x i16> %x, %add = mul <4 x i16> %y18, %y @@ -2186,6 +2557,54 @@ define <2 x i32> @clpeak_imad_pat_v2i32(<2 x i32> %x, <2 x i32> %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v3, v1 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i32: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v6, v0, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v6, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, v7, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[4:5], null, v2, v6, v[2:3] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[5:6], null, v3, v7, v[3:4] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, v[4:5] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[1:2], null, v5, v3, v[5:6] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i32: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v4, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v4 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v1, v3 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v3, v1 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <2 x i32> %x, %add = mul <2 x i32> %y18, %y @@ -2471,6 +2890,74 @@ define <3 x i32> @clpeak_imad_pat_v3i32(<3 x i32> %x, <3 x i32> %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v4, v1 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v2, v5, v2 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v3i32: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v2, 1, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v0, v3 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v8, v1, v4 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v9, v2, v5 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v7, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, v8, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v6, v9, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v3 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v1, v4 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v6, v5 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[5:6], null, v2, v7, v[2:3] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[6:7], null, v3, v8, v[3:4] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[7:8], null, v4, v9, v[4:5] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v5, v2, v[5:6] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[1:2], null, v6, v3, v[6:7] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[2:3], null, v7, v4, v[7:8] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v3i32: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v0, v3 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v7, v1, v4 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v8, v2, v5 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v6, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, v7, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, v8, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v3 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v4 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v2, v5 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v6 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v4, 1, v7 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v5, 1, v8 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v0, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v1, v4 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v2, v5 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v3, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v4, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v5, v2 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y48 = add <3 x i32> %x, %add = mul <3 x i32> %y48, %y @@ -2794,6 +3281,90 @@ define <4 x i32> @clpeak_imad_pat_v4i32(<4 x i32> %x, <4 x i32> %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u32 v2, v5, v2 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v3, v6, v3 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v4i32: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v2, 1, v2 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v3, 1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v8, v0, v4 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v9, v1, v5 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v10, v2, v6 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v11, v3, v7 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v8, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, v9, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v12, v10, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v4 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v11, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v1, v5 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v12, v6 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v0, v7 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[6:7], null, v2, v8, v[2:3] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[7:8], null, v3, v9, v[3:4] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[8:9], null, v4, v10, v[4:5] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[9:10], null, v5, v11, v[5:6] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v6, v2, v[6:7] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[1:2], null, v7, v3, v[7:8] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[2:3], null, v8, v4, v[8:9] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v9, v5, v[9:10] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v4i32: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v2 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v8, v0, v4 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v9, v1, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v10, v2, v6 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v11, v3, v7 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v8, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, v9, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, v10, v2 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, v11, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v4, 1, v8 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v5, 1, v9 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v6, 1, v10 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v7, 1, v11 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v8, 1, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v1, v5 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v2, v6 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v3, v7 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v2 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v3 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v8 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v4, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v5, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v6, v3 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <4 x i32> %x, %add = mul <4 x i32> %y18, %y @@ -2950,6 +3521,40 @@ define i32 @clpeak_imad_pat_i24(i32 %x, i32 %y) { ; GFX10-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i24: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GFX11-SDAG-NEXT: v_bfe_i32 v1, v1, 0, 24 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v1, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v5, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v2, v5, v[2:3] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v3, v2, v[3:4] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i24: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 24 +; GFX11-GISEL-NEXT: v_bfe_i32 v1, v1, 0, 24 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %shl = shl i32 %x, 8 %shr = ashr exact i32 %shl, 8 @@ -3110,6 +3715,40 @@ define i32 @clpeak_imad_pat_u24(i32 %x, i32 %y) { ; GFX10-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_u24: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; GFX11-SDAG-NEXT: v_and_b32_e32 v1, 0xffffff, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v1, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v5, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v2, v5, v[2:3] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v3, v2, v[3:4] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_u24: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffffff, v0 +; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffffff, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %shl = and i32 %x, 16777215 %shl1 = and i32 %y, 16777215 @@ -3240,6 +3879,35 @@ define signext i8 @clpeak_imad_pat_i8(i8 signext %x, i8 signext %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i8: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v0 +; GFX11-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 8 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i8: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v2, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v2, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 8 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %conv33 = add i8 %x, 1 %add = mul i8 %conv33, %y @@ -3447,6 +4115,60 @@ define <2 x i8> @clpeak_imad_pat_v2i8(<2 x i8> %x, <2 x i8> %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4 ; GFX10-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i8: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-SDAG-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u16 v4, v1, v3, v1 +; GFX11-SDAG-NEXT: v_mad_u16 v5, v0, v2, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u16 v1, v1, v3 +; GFX11-SDAG-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u16 v3, v4, v3 +; GFX11-SDAG-NEXT: v_mul_lo_u16 v2, v5, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v3, v1, v3 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v2, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v1, v3, v1 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v2, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_lshlrev_b16 v2, 8, v1 +; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0 +; GFX11-SDAG-NEXT: v_and_b32_e32 v1, 0xff, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_or_b32_e32 v0, v0, v2 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i8: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v4, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v4, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: v_add_nc_u16 v2, v4, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v3, v5, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add_nc_u16 v4, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v5, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <2 x i8> %x, %add = mul <2 x i8> %y18, %y @@ -3805,6 +4527,75 @@ define i64 @clpeak_imad_pat_i64(i64 %x, i64 %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u32 v3, v3, v4 ; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v2, v3 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i64: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_co_u32 v4, vcc_lo, v0, 1 +; GFX11-SDAG-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v4, v3 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, 0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v6, v5, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_add3_u32 v1, v1, v7, v6 +; GFX11-SDAG-NEXT: v_add_co_u32 v6, vcc_lo, v0, v4 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, v1, v5, vcc_lo +; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v6, v3 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v6, v2, 0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v5, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v3, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_add3_u32 v4, v4, v7, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v4, v0 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[5:6], null, v3, v0, v[3:4] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add3_u32 v6, v2, v6, v1 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v5, v4 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v6, v3 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v5, v3, v[5:6] +; GFX11-SDAG-NEXT: v_add3_u32 v1, v4, v1, v2 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i64: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v0, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v1, vcc_lo +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v4, v3 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v7, v5, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v6, v7 +; GFX11-GISEL-NEXT: v_add_co_u32 v6, vcc_lo, v0, v4 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, v1, v5, vcc_lo +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v6, v2, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v6, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v7, v2 +; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v4, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add3_u32 v5, v5, v3, v2 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v4, v0, 0 +; GFX11-GISEL-NEXT: v_add_co_u32 v4, vcc_lo, v4, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v5, v0 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v5, vcc_lo, 0, v5, vcc_lo +; GFX11-GISEL-NEXT: v_add3_u32 v3, v3, v1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v2, v4, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v2, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v3, v4 +; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v2, v3 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add i64 %x, 1 %add = mul i64 %y18, %y @@ -4458,6 +5249,120 @@ define <2 x i64> @clpeak_imad_pat_v2i64(<2 x i64> %x, <2 x i64> %y) { ; GFX10-GISEL-NEXT: v_add3_u32 v1, v1, v4, v7 ; GFX10-GISEL-NEXT: v_add3_u32 v3, v3, v6, v5 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i64: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_co_u32 v8, vcc_lo, v0, 1 +; GFX11-SDAG-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v1, vcc_lo +; GFX11-SDAG-NEXT: v_add_co_u32 v10, vcc_lo, v2, 1 +; GFX11-SDAG-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, 0, v3, vcc_lo +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v12, v9, v4 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v13, v8, v5 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v8, v4, 0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v14, v11, v6 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v15, v10, v7 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[2:3], null, v10, v6, 0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add3_u32 v1, v1, v13, v12 +; GFX11-SDAG-NEXT: v_add3_u32 v12, v3, v15, v14 +; GFX11-SDAG-NEXT: v_add_co_u32 v3, vcc_lo, v0, v8 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_add_co_ci_u32_e32 v8, vcc_lo, v1, v9, vcc_lo +; GFX11-SDAG-NEXT: v_add_co_u32 v9, vcc_lo, v2, v10 +; GFX11-SDAG-NEXT: v_add_co_ci_u32_e32 v10, vcc_lo, v12, v11, vcc_lo +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v11, v8, v4 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v13, v3, v5 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v14, v9, v7 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v10, v10, v6 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[7:8], null, v9, v6, 0 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[5:6], null, v3, v4, 0 +; GFX11-SDAG-NEXT: v_add3_u32 v8, v8, v14, v10 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add3_u32 v6, v6, v13, v11 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v11, v8, v2 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[9:10], null, v7, v2, v[7:8] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v13, v6, v0 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v5, v0, v[5:6] +; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v5, v1 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v7, v12 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v6, v3, v6 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3) +; GFX11-SDAG-NEXT: v_add3_u32 v4, v13, v4, v0 +; GFX11-SDAG-NEXT: v_add3_u32 v10, v11, v10, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v11, v4, v5 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v3, v5, v[3:4] +; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v9, v8 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v10, v7 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[2:3], null, v9, v7, v[9:10] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add3_u32 v1, v11, v1, v6 +; GFX11-SDAG-NEXT: v_add3_u32 v3, v5, v3, v4 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i64: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_co_u32 v8, vcc_lo, v0, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v1, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v10, vcc_lo, v2, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, 0, v3, vcc_lo +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v8, v4, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v12, v8, v5 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v13, v9, v4 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v10, v6, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v14, v10, v7 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v15, v11, v6 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v12, v13 +; GFX11-GISEL-NEXT: v_add3_u32 v12, v3, v14, v15 +; GFX11-GISEL-NEXT: v_add_co_u32 v3, vcc_lo, v0, v8 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v13, vcc_lo, v1, v9, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v10, vcc_lo, v2, v10 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, v12, v11, vcc_lo +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[8:9], null, v3, v4, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v3, v5 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v13, v13, v4 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[3:4], null, v10, v6, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v7, v10, v7 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v11, v6 +; GFX11-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v0, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v1, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v2, vcc_lo, v2, 1 +; GFX11-GISEL-NEXT: v_add3_u32 v9, v9, v5, v13 +; GFX11-GISEL-NEXT: v_add3_u32 v10, v4, v7, v6 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v11, vcc_lo, 0, v12, vcc_lo +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v8, v0, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v8, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v9, v0 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v3, v2, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v11, v3, v11 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v10, v2 +; GFX11-GISEL-NEXT: v_add_co_u32 v8, vcc_lo, v8, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v9, vcc_lo, 0, v9, vcc_lo +; GFX11-GISEL-NEXT: v_add_co_u32 v12, vcc_lo, v3, 1 +; GFX11-GISEL-NEXT: v_add_co_ci_u32_e32 v10, vcc_lo, 0, v10, vcc_lo +; GFX11-GISEL-NEXT: v_add3_u32 v3, v5, v1, v0 +; GFX11-GISEL-NEXT: v_add3_u32 v5, v7, v11, v2 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v4, v8, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v4, v9 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_4) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v7, v3, v8 +; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v6, v12, 0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v6, v10 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v5, v12 +; GFX11-GISEL-NEXT: v_add3_u32 v1, v1, v4, v7 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add3_u32 v3, v3, v6, v5 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y18 = add <2 x i64> %x, %add = mul <2 x i64> %y18, %y @@ -4577,23 +5482,41 @@ define i32 @v_multi_use_mul_chain_add_other_use_all(i32 %arg, i32 %arg1, i32 %ar ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v5, v1 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; -; GFX900-LABEL: v_multi_use_mul_chain_add_other_use_all: -; GFX900: ; %bb.0: ; %bb -; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX900-NEXT: v_add_u32_e32 v0, 1, v0 -; GFX900-NEXT: v_mul_lo_u32 v2, v0, v1 -; GFX900-NEXT: v_add_u32_e32 v0, v2, v0 -; GFX900-NEXT: v_mul_lo_u32 v0, v0, v1 -; GFX900-NEXT: v_add_u32_e32 v1, 1, v2 -; GFX900-NEXT: v_mul_lo_u32 v5, v0, v1 -; GFX900-NEXT: global_store_dword v[3:4], v2, off -; GFX900-NEXT: s_waitcnt vmcnt(0) -; GFX900-NEXT: global_store_dword v[3:4], v0, off -; GFX900-NEXT: s_waitcnt vmcnt(0) -; GFX900-NEXT: global_store_dword v[3:4], v5, off -; GFX900-NEXT: s_waitcnt vmcnt(0) -; GFX900-NEXT: v_add_u32_e32 v0, v5, v1 -; GFX900-NEXT: s_setpc_b64 s[30:31] +; GFX900-SDAG-LABEL: v_multi_use_mul_chain_add_other_use_all: +; GFX900-SDAG: ; %bb.0: ; %bb +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-SDAG-NEXT: v_add_u32_e32 v0, 1, v0 +; GFX900-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX900-SDAG-NEXT: v_add_u32_e32 v0, v2, v0 +; GFX900-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX900-SDAG-NEXT: v_add_u32_e32 v1, 1, v2 +; GFX900-SDAG-NEXT: v_mul_lo_u32 v5, v0, v1 +; GFX900-SDAG-NEXT: global_store_dword v[3:4], v2, off +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX900-SDAG-NEXT: global_store_dword v[3:4], v0, off +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX900-SDAG-NEXT: global_store_dword v[3:4], v5, off +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX900-SDAG-NEXT: v_add_u32_e32 v0, v5, v1 +; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX900-GISEL-LABEL: v_multi_use_mul_chain_add_other_use_all: +; GFX900-GISEL: ; %bb.0: ; %bb +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-GISEL-NEXT: v_add_u32_e32 v0, 1, v0 +; GFX900-GISEL-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX900-GISEL-NEXT: v_add_u32_e32 v0, v2, v0 +; GFX900-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX900-GISEL-NEXT: v_add_u32_e32 v1, 1, v2 +; GFX900-GISEL-NEXT: v_mul_lo_u32 v5, v0, v1 +; GFX900-GISEL-NEXT: global_store_dword v[3:4], v2, off +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX900-GISEL-NEXT: global_store_dword v[3:4], v0, off +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX900-GISEL-NEXT: global_store_dword v[3:4], v5, off +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX900-GISEL-NEXT: v_add_u32_e32 v0, v5, v1 +; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31] ; ; GFX90A-SDAG-LABEL: v_multi_use_mul_chain_add_other_use_all: ; GFX90A-SDAG: ; %bb.0: ; %bb @@ -4652,6 +5575,26 @@ define i32 @v_multi_use_mul_chain_add_other_use_all(i32 %arg, i32 %arg1, i32 %ar ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX10-NEXT: v_add_nc_u32_e32 v0, v5, v0 ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: v_multi_use_mul_chain_add_other_use_all: +; GFX11: ; %bb.0: ; %bb +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-NEXT: v_add_nc_u32_e32 v0, 1, v2 +; GFX11-NEXT: v_mul_lo_u32 v5, v1, v0 +; GFX11-NEXT: global_store_b32 v[3:4], v2, off dlc +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_store_b32 v[3:4], v1, off dlc +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_store_b32 v[3:4], v5, off dlc +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_add_nc_u32_e32 v0, v5, v0 +; GFX11-NEXT: s_setpc_b64 s[30:31] bb: %i = add i32 %arg, 1 %i3 = mul i32 %i, %arg1 @@ -4761,21 +5704,37 @@ define i32 @v_multi_use_mul_chain_add_other_use_some(i32 %arg, i32 %arg1, i32 %a ; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v1 ; GFX8-NEXT: s_setpc_b64 s[30:31] ; -; GFX900-LABEL: v_multi_use_mul_chain_add_other_use_some: -; GFX900: ; %bb.0: ; %bb -; GFX900-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) -; GFX900-NEXT: v_add_u32_e32 v0, 1, v0 -; GFX900-NEXT: v_mul_lo_u32 v2, v0, v1 -; GFX900-NEXT: v_add_u32_e32 v0, v2, v0 -; GFX900-NEXT: v_mul_lo_u32 v0, v0, v1 -; GFX900-NEXT: v_add_u32_e32 v1, 1, v2 -; GFX900-NEXT: v_mul_lo_u32 v0, v0, v1 -; GFX900-NEXT: global_store_dword v[3:4], v2, off -; GFX900-NEXT: s_waitcnt vmcnt(0) -; GFX900-NEXT: global_store_dword v[3:4], v0, off -; GFX900-NEXT: s_waitcnt vmcnt(0) -; GFX900-NEXT: v_add_u32_e32 v0, v0, v1 -; GFX900-NEXT: s_setpc_b64 s[30:31] +; GFX900-SDAG-LABEL: v_multi_use_mul_chain_add_other_use_some: +; GFX900-SDAG: ; %bb.0: ; %bb +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-SDAG-NEXT: v_add_u32_e32 v0, 1, v0 +; GFX900-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX900-SDAG-NEXT: v_add_u32_e32 v0, v2, v0 +; GFX900-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX900-SDAG-NEXT: v_add_u32_e32 v1, 1, v2 +; GFX900-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX900-SDAG-NEXT: global_store_dword v[3:4], v2, off +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX900-SDAG-NEXT: global_store_dword v[3:4], v0, off +; GFX900-SDAG-NEXT: s_waitcnt vmcnt(0) +; GFX900-SDAG-NEXT: v_add_u32_e32 v0, v0, v1 +; GFX900-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX900-GISEL-LABEL: v_multi_use_mul_chain_add_other_use_some: +; GFX900-GISEL: ; %bb.0: ; %bb +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX900-GISEL-NEXT: v_add_u32_e32 v0, 1, v0 +; GFX900-GISEL-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX900-GISEL-NEXT: v_add_u32_e32 v0, v2, v0 +; GFX900-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX900-GISEL-NEXT: v_add_u32_e32 v1, 1, v2 +; GFX900-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX900-GISEL-NEXT: global_store_dword v[3:4], v2, off +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX900-GISEL-NEXT: global_store_dword v[3:4], v0, off +; GFX900-GISEL-NEXT: s_waitcnt vmcnt(0) +; GFX900-GISEL-NEXT: v_add_u32_e32 v0, v0, v1 +; GFX900-GISEL-NEXT: s_setpc_b64 s[30:31] ; ; GFX90A-SDAG-LABEL: v_multi_use_mul_chain_add_other_use_some: ; GFX90A-SDAG: ; %bb.0: ; %bb @@ -4828,6 +5787,24 @@ define i32 @v_multi_use_mul_chain_add_other_use_some(i32 %arg, i32 %arg1, i32 %a ; GFX10-NEXT: s_waitcnt_vscnt null, 0x0 ; GFX10-NEXT: v_add_nc_u32_e32 v0, v5, v1 ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: v_multi_use_mul_chain_add_other_use_some: +; GFX11: ; %bb.0: ; %bb +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX11-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-NEXT: v_mul_lo_u32 v5, v0, v1 +; GFX11-NEXT: global_store_b32 v[3:4], v2, off dlc +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: global_store_b32 v[3:4], v5, off dlc +; GFX11-NEXT: s_waitcnt_vscnt null, 0x0 +; GFX11-NEXT: v_add_nc_u32_e32 v0, v5, v1 +; GFX11-NEXT: s_setpc_b64 s[30:31] bb: %i = add i32 %arg, 1 %i3 = mul i32 %i, %arg1 @@ -5045,6 +6022,58 @@ define i32 @clpeak_imad_pat_i32_x2(i32 %x, i32 %y) { ; GFX10-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i32_x2: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, v2, v1 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v0, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, v5, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v2, v5, v[2:3] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v3, v2, v[3:4] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i32_x2: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v2, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, v2, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, v2, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v1, v0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y38 = add i32 %x, 1 %add = mul i32 %y38, %y @@ -5424,6 +6453,98 @@ define <2 x i32> @clpeak_imad_pat_v2i32_x2(<2 x i32> %x, <2 x i32> %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0 ; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v3, v1 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i32_x2: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v0, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v0, v4, v0 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v1, v5, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v2, 1, v4 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v3, 1, v5 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v0, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v2, v4, v2 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v3, v5, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v2, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v3, v1 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v2, 1, v4 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v3, 1, v5 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v6, v0, v2 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v2, v6, v2 +; GFX11-SDAG-NEXT: v_add_nc_u32_e32 v3, v7, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v2, v0 +; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v3, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[4:5], null, v2, v6, v[2:3] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[5:6], null, v3, v7, v[3:4] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, v[4:5] +; GFX11-SDAG-NEXT: v_mad_u64_u32 v[1:2], null, v5, v3, v[5:6] +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i32_x2: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, v4, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, v5, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v4 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, v4, v2 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, v5, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v3, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v4 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v1, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, v4, v2 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, v5, v3 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v3, v1 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v2, 1, v4 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v3, 1, v5 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v0, v2 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v1, v3 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v0, 1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u32_e32 v1, 1, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v2, v0 +; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v3, v1 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y38 = add <2 x i32> %x, %add = mul <2 x i32> %y38, %y @@ -5628,6 +6749,51 @@ define signext i16 @clpeak_imad_pat_i16_x2(i16 signext %x, i16 signext %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 16 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_i16_x2: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v0 +; GFX11-SDAG-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_i16_x2: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v2, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v2, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_bfe_i32 v0, v0, 0, 16 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %conv69 = add i16 %x, 1 %add = mul i16 %conv69, %y @@ -5826,6 +6992,51 @@ define zeroext i16 @clpeak_umad_pat_i16_x2(i16 zeroext %x, i16 zeroext %y) { ; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_umad_pat_i16_x2: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v1, v0, v1, v0 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v1, v0, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v0 +; GFX11-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_umad_pat_i16_x2: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v2, v1, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v2, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v0 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, 1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v1, 1 +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %conv69 = add i16 %x, 1 %add = mul i16 %conv69, %y @@ -6149,6 +7360,60 @@ define <2 x i16> @clpeak_imad_pat_v2i16_x2(<2 x i16> %x, <2 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_imad_pat_v2i16_x2: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v2, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_add_u16 v2, v1, v2 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v1, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v2, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v2, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v1, v0 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v2, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_imad_pat_v2i16_x2: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v2, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v0, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v1, v2 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v2, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v2, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v1, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y38 = add <2 x i16> %x, %add = mul <2 x i16> %y38, %y @@ -6472,6 +7737,60 @@ define <2 x i16> @clpeak_umad_pat_v2i16_x2(<2 x i16> %x, <2 x i16> %y) { ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 ; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: clpeak_umad_pat_v2i16_x2: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_pk_sub_u16 v0, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v2, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v1, v0, v2 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_add_u16 v2, v1, v2 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v1, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v2, v0 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-SDAG-NEXT: v_pk_add_u16 v1, v2, v1 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v1, v0 +; GFX11-SDAG-NEXT: v_pk_sub_u16 v1, v2, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: v_pk_sub_u16 v2, v0, -1 op_sel_hi:[1,0] +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: clpeak_umad_pat_v2i16_x2: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v0, v2, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v2, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v0, v2 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v1, v2 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v1, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v2, v0 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v0, v1 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v2, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v1, v0 +; GFX11-GISEL-NEXT: v_pk_add_u16 v1, v2, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: v_pk_add_u16 v2, v0, 1 op_sel_hi:[1,0] +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %y38 = add <2 x i16> %x, %add = mul <2 x i16> %y38, %y @@ -6525,6 +7844,15 @@ define <2 x i32> @multi_use_mul_mad_i32_var(i32 %x, i32 %y, i32 %z0, i32 %z1) { ; GFX10-NEXT: v_add_nc_u32_e32 v0, v1, v2 ; GFX10-NEXT: v_add_nc_u32_e32 v1, v1, v3 ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: multi_use_mul_mad_i32_var: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_add_nc_u32_e32 v0, v1, v2 +; GFX11-NEXT: v_add_nc_u32_e32 v1, v1, v3 +; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %mul = mul i32 %x, %y %add0 = add i32 %mul, %z0 @@ -6606,6 +7934,27 @@ define <2 x i16> @multi_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z0, i16 %z1) { ; GFX10-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1 ; GFX10-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1 ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: multi_use_mul_mad_i16_var: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mad_u16 v2, v0, v1, v2 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v3 +; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-SDAG-NEXT: v_perm_b32 v0, v0, v2, 0x5040100 +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: multi_use_mul_mad_i16_var: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2) +; GFX11-GISEL-NEXT: v_add_nc_u16 v1, v0, v2 +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v0, v3 +; GFX11-GISEL-NEXT: v_and_b32_e32 v1, 0xffff, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_lshl_or_b32 v0, v0, 16, v1 +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %mul = mul i16 %x, %y %add0 = add i16 %mul, %z0 @@ -6653,6 +8002,16 @@ define i32 @other_use_mul_mad_i32_var(i32 %x, i32 %y, i32 %z, ptr addrspace(3) % ; GFX10-NEXT: ds_write_b32 v3, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: other_use_mul_mad_i32_var: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_mul_lo_u32 v1, v0, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_add_nc_u32_e32 v0, v1, v2 +; GFX11-NEXT: ds_store_b32 v3, v1 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %mul = mul i32 %x, %y %add0 = add i32 %mul, %z @@ -6721,6 +8080,25 @@ define i16 @other_use_mul_mad_i16_var(i16 %x, i16 %y, i16 %z, ptr addrspace(3) % ; GFX10-GISEL-NEXT: ds_write_b16 v3, v1 ; GFX10-GISEL-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-SDAG-LABEL: other_use_mul_mad_i16_var: +; GFX11-SDAG: ; %bb.0: ; %entry +; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-SDAG-NEXT: v_mul_lo_u16 v4, v0, v1 +; GFX11-SDAG-NEXT: v_mad_u16 v0, v0, v1, v2 +; GFX11-SDAG-NEXT: ds_store_b16 v3, v4 +; GFX11-SDAG-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-GISEL-LABEL: other_use_mul_mad_i16_var: +; GFX11-GISEL: ; %bb.0: ; %entry +; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v0, v1 +; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-GISEL-NEXT: v_add_nc_u16 v0, v1, v2 +; GFX11-GISEL-NEXT: ds_store_b16 v3, v1 +; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31] entry: %mul = mul i16 %x, %y %add0 = add i16 %mul, %z @@ -6816,6 +8194,15 @@ define <4 x i16> @multi_use_mul_mad_v2i16_var(<2 x i16> %x, <2 x i16> %y, <2 x i ; GFX10-NEXT: v_pk_add_u16 v0, v1, v2 ; GFX10-NEXT: v_pk_add_u16 v1, v1, v3 ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: multi_use_mul_mad_v2i16_var: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_pk_mul_lo_u16 v1, v0, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_pk_add_u16 v0, v1, v2 +; GFX11-NEXT: v_pk_add_u16 v1, v1, v3 +; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %mul = mul <2 x i16> %x, %y %add0 = add <2 x i16> %mul, %z0 @@ -6921,6 +8308,16 @@ define <2 x i16> @other_use_mul_mad_v2i16_var(<2 x i16> %x, <2 x i16> %y, <2 x i ; GFX10-NEXT: ds_write_b32 v3, v1 ; GFX10-NEXT: s_waitcnt lgkmcnt(0) ; GFX10-NEXT: s_setpc_b64 s[30:31] +; +; GFX11-LABEL: other_use_mul_mad_v2i16_var: +; GFX11: ; %bb.0: ; %entry +; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX11-NEXT: v_pk_mul_lo_u16 v1, v0, v1 +; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) +; GFX11-NEXT: v_pk_add_u16 v0, v1, v2 +; GFX11-NEXT: ds_store_b32 v3, v1 +; GFX11-NEXT: s_waitcnt lgkmcnt(0) +; GFX11-NEXT: s_setpc_b64 s[30:31] entry: %mul = mul <2 x i16> %x, %y %add0 = add <2 x i16> %mul, %z @@ -6928,7 +8325,102 @@ entry: ret <2 x i16> %add0 } +define i64 @mul_u24_add64(i32 %x, i32 %y, i64 %z) { +; GFX67-LABEL: mul_u24_add64: +; GFX67: ; %bb.0: +; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX67-NEXT: v_mul_hi_u32_u24_e32 v4, v0, v1 +; GFX67-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX67-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX67-NEXT: v_addc_u32_e32 v1, vcc, v4, v3, vcc +; GFX67-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-LABEL: mul_u24_add64: +; GFX8: ; %bb.0: +; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: v_mul_hi_u32_u24_e32 v4, v0, v1 +; GFX8-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2 +; GFX8-NEXT: v_addc_u32_e32 v1, vcc, v4, v3, vcc +; GFX8-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-SDAG-LABEL: mul_u24_add64: +; GFX9-SDAG: ; %bb.0: +; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v1, v[2:3] +; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-GISEL-LABEL: mul_u24_add64: +; GFX9-GISEL: ; %bb.0: +; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-GISEL-NEXT: v_mul_hi_u32_u24_e32 v4, v0, v1 +; GFX9-GISEL-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX9-GISEL-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 +; GFX9-GISEL-NEXT: v_addc_co_u32_e32 v1, vcc, v4, v3, vcc +; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-SDAG-LABEL: mul_u24_add64: +; GFX10-SDAG: ; %bb.0: +; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v0, v1, v[2:3] +; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-GISEL-LABEL: mul_u24_add64: +; GFX10-GISEL: ; %bb.0: +; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-GISEL-NEXT: v_mul_u32_u24_e32 v4, v0, v1 +; GFX10-GISEL-NEXT: v_mul_hi_u32_u24_e32 v1, v0, v1 +; GFX10-GISEL-NEXT: v_add_co_u32 v0, vcc_lo, v4, v2 +; GFX10-GISEL-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, v1, v3, vcc_lo +; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31] + %mul = call i64 @llvm.amdgcn.mul.u24.i64(i32 %x, i32 %y) + %add = add i64 %mul, %z + ret i64 %add +} + +define i64 @mul_u24_zext_add64(i32 %x, i32 %y, i64 %z) { +; GFX67-LABEL: mul_u24_zext_add64: +; GFX67: ; %bb.0: +; GFX67-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX67-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX67-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX67-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc +; GFX67-NEXT: s_setpc_b64 s[30:31] +; +; GFX8-LABEL: mul_u24_zext_add64: +; GFX8: ; %bb.0: +; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX8-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX8-NEXT: v_add_u32_e32 v0, vcc, v0, v2 +; GFX8-NEXT: v_addc_u32_e32 v1, vcc, 0, v3, vcc +; GFX8-NEXT: s_setpc_b64 s[30:31] +; +; GFX9-LABEL: mul_u24_zext_add64: +; GFX9: ; %bb.0: +; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX9-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v2 +; GFX9-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v3, vcc +; GFX9-NEXT: s_setpc_b64 s[30:31] +; +; GFX10-LABEL: mul_u24_zext_add64: +; GFX10: ; %bb.0: +; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GFX10-NEXT: v_mul_u32_u24_e32 v0, v0, v1 +; GFX10-NEXT: v_add_co_u32 v0, vcc_lo, v0, v2 +; GFX10-NEXT: v_add_co_ci_u32_e32 v1, vcc_lo, 0, v3, vcc_lo +; GFX10-NEXT: s_setpc_b64 s[30:31] + %mul = call i32 @llvm.amdgcn.mul.u24(i32 %x, i32 %y) + %mul.zext = zext i32 %mul to i64 + %add = add i64 %mul.zext, %z + ret i64 %add +} + +declare i64 @llvm.amdgcn.mul.u24.i64(i32, i32) +declare i32 @llvm.amdgcn.mul.u24(i32, i32) + ;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: ; GFX6: {{.*}} ; GFX7: {{.*}} +; GFX900: {{.*}} ; GFX90A: {{.*}} diff --git a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll index d9c6fbb31901..cf5886010160 100644 --- a/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll +++ b/llvm/test/CodeGen/AMDGPU/machine-sink-temporal-divergence-swdev407790.ll @@ -444,31 +444,28 @@ define protected amdgpu_kernel void @kernel_round1(ptr addrspace(1) nocapture no ; CHECK-NEXT: s_xor_b32 s4, exec_lo, s4 ; CHECK-NEXT: s_cbranch_execz .LBB0_31 ; CHECK-NEXT: ; %bb.30: ; in Loop: Header=BB0_28 Depth=1 -; CHECK-NEXT: v_xor_b32_e32 v5, v60, v58 -; CHECK-NEXT: v_lshrrev_b64 v[3:4], 16, v[56:57] -; CHECK-NEXT: v_mul_u32_u24_e32 v11, 0x180, v73 -; CHECK-NEXT: v_lshlrev_b32_e32 v0, 5, v0 -; CHECK-NEXT: v_lshrrev_b64 v[1:2], 16, v[45:46] -; CHECK-NEXT: v_lshlrev_b32_e32 v7, 16, v5 +; CHECK-NEXT: v_xor_b32_e32 v4, v60, v58 +; CHECK-NEXT: v_lshrrev_b64 v[2:3], 16, v[56:57] +; CHECK-NEXT: v_mad_u64_u32 v[6:7], null, 0x180, v73, s[46:47] +; CHECK-NEXT: v_lshlrev_b32_e32 v10, 5, v0 +; CHECK-NEXT: v_lshlrev_b32_e32 v1, 16, v4 ; CHECK-NEXT: v_lshlrev_b32_e32 v8, 6, v72 -; CHECK-NEXT: v_add_co_u32 v11, vcc_lo, s46, v11 -; CHECK-NEXT: v_lshlrev_b32_e32 v10, 12, v63 -; CHECK-NEXT: v_or_b32_e32 v4, v7, v4 -; CHECK-NEXT: v_mul_hi_u32_u24_e32 v7, 0x180, v73 -; CHECK-NEXT: v_xor_b32_e32 v6, v61, v59 -; CHECK-NEXT: v_lshlrev_b32_e32 v9, 16, v56 -; CHECK-NEXT: v_or3_b32 v10, v8, v10, v62 +; CHECK-NEXT: v_lshlrev_b32_e32 v9, 12, v63 +; CHECK-NEXT: v_xor_b32_e32 v5, v61, v59 +; CHECK-NEXT: v_lshlrev_b32_e32 v11, 16, v56 +; CHECK-NEXT: v_or_b32_e32 v3, v1, v3 +; CHECK-NEXT: v_lshrrev_b64 v[0:1], 16, v[45:46] +; CHECK-NEXT: v_add_co_u32 v6, vcc_lo, v6, v10 +; CHECK-NEXT: v_or3_b32 v8, v8, v9, v62 +; CHECK-NEXT: v_add_co_ci_u32_e32 v7, vcc_lo, 0, v7, vcc_lo +; CHECK-NEXT: v_lshrrev_b64 v[4:5], 16, v[4:5] +; CHECK-NEXT: v_or_b32_e32 v1, v11, v1 ; CHECK-NEXT: ; implicit-def: $vgpr42 ; CHECK-NEXT: ; implicit-def: $vgpr43 ; CHECK-NEXT: ; implicit-def: $vgpr44 -; CHECK-NEXT: v_add_co_ci_u32_e32 v12, vcc_lo, s47, v7, vcc_lo -; CHECK-NEXT: v_add_co_u32 v7, vcc_lo, v11, v0 -; CHECK-NEXT: v_lshrrev_b64 v[5:6], 16, v[5:6] -; CHECK-NEXT: v_add_co_ci_u32_e32 v8, vcc_lo, 0, v12, vcc_lo -; CHECK-NEXT: v_or_b32_e32 v2, v9, v2 -; CHECK-NEXT: global_store_dword v[7:8], v10, off offset:4 -; CHECK-NEXT: global_store_dwordx4 v[7:8], v[1:4], off offset:8 -; CHECK-NEXT: global_store_dwordx2 v[7:8], v[5:6], off offset:24 +; CHECK-NEXT: global_store_dword v[6:7], v8, off offset:4 +; CHECK-NEXT: global_store_dwordx4 v[6:7], v[0:3], off offset:8 +; CHECK-NEXT: global_store_dwordx2 v[6:7], v[4:5], off offset:24 ; CHECK-NEXT: .LBB0_31: ; %Flow ; CHECK-NEXT: ; in Loop: Header=BB0_28 Depth=1 ; CHECK-NEXT: s_andn2_saveexec_b32 s4, s4 diff --git a/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir b/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir index 83e85ccf7f8f..f0c5ba489ef3 100644 --- a/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir +++ b/llvm/test/CodeGen/AMDGPU/regpressure_printer.mir @@ -47,87 +47,46 @@ body: | name: live_through_test tracksRegLiveness: true body: | - ; RPU-LABEL: name: live_through_test - ; RPU: bb.0: - ; RPU-NEXT: Live-in: - ; RPU-NEXT: SGPR VGPR - ; RPU-NEXT: 0 0 - ; RPU-NEXT: 3 0 %0:sgpr_128 = IMPLICIT_DEF - ; RPU-NEXT: 3 0 - ; RPU-NEXT: Live-out: %0:00000000000000F3 - ; RPU-NEXT: Live-thr: - ; RPU-NEXT: 0 0 - ; RPU-NEXT: bb.1: - ; RPU-NEXT: Live-in: %0:00000000000000F3 - ; RPU-NEXT: SGPR VGPR - ; RPU-NEXT: 3 0 - ; RPU-NEXT: 3 0 S_NOP 0, implicit %0.sub0:sgpr_128 - ; RPU-NEXT: 2 0 - ; RPU-NEXT: 3 0 %0.sub0:sgpr_128 = IMPLICIT_DEF - ; RPU-NEXT: 3 0 - ; RPU-NEXT: 3 0 %0.sub1:sgpr_128 = IMPLICIT_DEF - ; RPU-NEXT: 3 0 - ; RPU-NEXT: 3 0 S_NOP 0, implicit %0.sub2:sgpr_128 - ; RPU-NEXT: 2 0 - ; RPU-NEXT: 3 0 %0.sub2:sgpr_128 = IMPLICIT_DEF - ; RPU-NEXT: 3 0 - ; RPU-NEXT: 3 0 S_NOP 0, implicit %0.sub2:sgpr_128 - ; RPU-NEXT: 2 0 - ; RPU-NEXT: 2 0 S_NOP 0, implicit %0.sub3:sgpr_128 - ; RPU-NEXT: 2 0 - ; RPU-NEXT: Live-out: %0:00000000000000C3 - ; RPU-NEXT: Live-thr: %0:00000000000000C0 - ; RPU-NEXT: 1 0 - ; RPU-NEXT: bb.2: - ; RPU-NEXT: Live-in: %0:00000000000000C3 - ; RPU-NEXT: SGPR VGPR - ; RPU-NEXT: 2 0 - ; RPU-NEXT: 2 0 S_NOP 0, implicit %0.sub3:sgpr_128, implicit %0.sub0:sgpr_128 - ; RPU-NEXT: 0 0 - ; RPU-NEXT: Live-out: - ; RPU-NEXT: Live-thr: - ; RPU-NEXT: 0 0 - ; - ; RPD-LABEL: name: live_through_test - ; RPD: bb.0: - ; RPD-NEXT: Live-in: - ; RPD-NEXT: SGPR VGPR - ; RPD-NEXT: 0 0 - ; RPD-NEXT: 4 0 %0:sgpr_128 = IMPLICIT_DEF - ; RPD-NEXT: 3 0 - ; RPD-NEXT: Live-out: %0:00000000000000F3 - ; RPD-NEXT: Live-thr: - ; RPD-NEXT: 0 0 - ; RPD-NEXT: bb.1: - ; RPD-NEXT: Live-in: %0:00000000000000F3 - ; RPD-NEXT: SGPR VGPR - ; RPD-NEXT: 3 0 - ; RPD-NEXT: 3 0 S_NOP 0, implicit %0.sub0:sgpr_128 - ; RPD-NEXT: 2 0 - ; RPD-NEXT: 3 0 %0.sub0:sgpr_128 = IMPLICIT_DEF - ; RPD-NEXT: 3 0 - ; RPD-NEXT: 4 0 %0.sub1:sgpr_128 = IMPLICIT_DEF - ; RPD-NEXT: 3 0 - ; RPD-NEXT: 3 0 S_NOP 0, implicit %0.sub2:sgpr_128 - ; RPD-NEXT: 2 0 - ; RPD-NEXT: 3 0 %0.sub2:sgpr_128 = IMPLICIT_DEF - ; RPD-NEXT: 3 0 - ; RPD-NEXT: 3 0 S_NOP 0, implicit %0.sub2:sgpr_128 - ; RPD-NEXT: 2 0 - ; RPD-NEXT: 2 0 S_NOP 0, implicit %0.sub3:sgpr_128 - ; RPD-NEXT: 2 0 - ; RPD-NEXT: Live-out: %0:00000000000000C3 - ; RPD-NEXT: Live-thr: %0:00000000000000C0 - ; RPD-NEXT: 1 0 - ; RPD-NEXT: bb.2: - ; RPD-NEXT: Live-in: %0:00000000000000C3 - ; RPD-NEXT: SGPR VGPR - ; RPD-NEXT: 2 0 - ; RPD-NEXT: 2 0 S_NOP 0, implicit %0.sub3:sgpr_128, implicit %0.sub0:sgpr_128 - ; RPD-NEXT: 0 0 - ; RPD-NEXT: Live-out: - ; RPD-NEXT: Live-thr: - ; RPD-NEXT: 0 0 + ; RP-LABEL: name: live_through_test + ; RP: bb.0: + ; RP-NEXT: Live-in: + ; RP-NEXT: SGPR VGPR + ; RP-NEXT: 0 0 + ; RP-NEXT: 4 0 %0:sgpr_128 = IMPLICIT_DEF + ; RP-NEXT: 3 0 + ; RP-NEXT: Live-out: %0:00000000000000F3 + ; RP-NEXT: Live-thr: + ; RP-NEXT: 0 0 + ; RP-NEXT: bb.1: + ; RP-NEXT: Live-in: %0:00000000000000F3 + ; RP-NEXT: SGPR VGPR + ; RP-NEXT: 3 0 + ; RP-NEXT: 3 0 S_NOP 0, implicit %0.sub0:sgpr_128 + ; RP-NEXT: 2 0 + ; RP-NEXT: 3 0 %0.sub0:sgpr_128 = IMPLICIT_DEF + ; RP-NEXT: 3 0 + ; RP-NEXT: 4 0 %0.sub1:sgpr_128 = IMPLICIT_DEF + ; RP-NEXT: 3 0 + ; RP-NEXT: 3 0 S_NOP 0, implicit %0.sub2:sgpr_128 + ; RP-NEXT: 2 0 + ; RP-NEXT: 3 0 %0.sub2:sgpr_128 = IMPLICIT_DEF + ; RP-NEXT: 3 0 + ; RP-NEXT: 3 0 S_NOP 0, implicit %0.sub2:sgpr_128 + ; RP-NEXT: 2 0 + ; RP-NEXT: 2 0 S_NOP 0, implicit %0.sub3:sgpr_128 + ; RP-NEXT: 2 0 + ; RP-NEXT: Live-out: %0:00000000000000C3 + ; RP-NEXT: Live-thr: %0:00000000000000C0 + ; RP-NEXT: 1 0 + ; RP-NEXT: bb.2: + ; RP-NEXT: Live-in: %0:00000000000000C3 + ; RP-NEXT: SGPR VGPR + ; RP-NEXT: 2 0 + ; RP-NEXT: 2 0 S_NOP 0, implicit %0.sub3:sgpr_128, implicit %0.sub0:sgpr_128 + ; RP-NEXT: 0 0 + ; RP-NEXT: Live-out: + ; RP-NEXT: Live-thr: + ; RP-NEXT: 0 0 bb.0: %0:sgpr_128 = IMPLICIT_DEF bb.1: @@ -223,7 +182,7 @@ body: | ; RPU-NEXT: 0 7 ; RPU-NEXT: 0 7 %7:vgpr_32 = GLOBAL_LOAD_DWORD %5:vreg_64, 0, 0, implicit $exec ; RPU-NEXT: 0 6 - ; RPU-NEXT: 0 7 %8:vreg_64 = IMPLICIT_DEF + ; RPU-NEXT: 0 8 %8:vreg_64 = IMPLICIT_DEF ; RPU-NEXT: 0 7 ; RPU-NEXT: 0 9 %9:vreg_64 = IMPLICIT_DEF ; RPU-NEXT: 0 9 @@ -262,7 +221,7 @@ body: | ; RPU-NEXT: 0 12 ; RPU-NEXT: 0 12 dead %21:vgpr_32 = GLOBAL_LOAD_DWORD %14:vreg_64, 0, 0, implicit $exec ; RPU-NEXT: 0 10 - ; RPU-NEXT: 0 10 dead %22:vgpr_32 = GLOBAL_LOAD_DWORD %15:vreg_64, 0, 0, implicit $exec + ; RPU-NEXT: 0 11 dead %22:vgpr_32 = GLOBAL_LOAD_DWORD %15:vreg_64, 0, 0, implicit $exec ; RPU-NEXT: 0 10 ; RPU-NEXT: 0 10 %23:vreg_64 = V_LSHLREV_B64_e64 2, %8:vreg_64, implicit $exec ; RPU-NEXT: 0 9 @@ -550,7 +509,7 @@ body: | ; RPU-NEXT: 0 0 ; RPU-NEXT: 0 0 $sgpr0 = S_BUFFER_LOAD_DWORD_IMM $sgpr0_sgpr1_sgpr2_sgpr3, 0, 0 ; RPU-NEXT: 0 0 - ; RPU-NEXT: 0 0 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec + ; RPU-NEXT: 0 1 undef %0.sub5:vreg_512 = V_MOV_B32_e32 5, implicit $exec ; RPU-NEXT: 0 0 ; RPU-NEXT: 0 0 S_CMP_GT_U32 $sgpr0, 15, implicit-def $scc ; RPU-NEXT: 0 0 @@ -569,7 +528,7 @@ body: | ; RPU-NEXT: 0 1 ; RPU-NEXT: 0 1 $m0 = S_MOV_B32 killed $sgpr0 ; RPU-NEXT: 0 1 - ; RPU-NEXT: 0 1 %0:vreg_512 = V_INDIRECT_REG_WRITE_MOVREL_B32_V16 %0:vreg_512(tied-def 0), 42, 3, implicit $m0, implicit $exec + ; RPU-NEXT: 0 16 %0:vreg_512 = V_INDIRECT_REG_WRITE_MOVREL_B32_V16 %0:vreg_512(tied-def 0), 42, 3, implicit $m0, implicit $exec ; RPU-NEXT: 0 1 ; RPU-NEXT: Live-out: %0:0000000000000C00 ; RPU-NEXT: Live-thr: @@ -709,33 +668,19 @@ tracksRegLiveness: true body: | bb.0: liveins: $sgpr0_sgpr1_sgpr2_sgpr3 - ; RPU-LABEL: name: test_partially_used_early_clobber_def - ; RPU: Live-in: - ; RPU-NEXT: SGPR VGPR - ; RPU-NEXT: 0 0 - ; RPU-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3 - ; RPU-NEXT: 4 0 - ; RPU-NEXT: 5 0 early-clobber %1:sgpr_128 = COPY %0:sgpr_128 - ; RPU-NEXT: 1 0 - ; RPU-NEXT: 1 0 S_NOP 0, implicit %1.sub1:sgpr_128 - ; RPU-NEXT: 0 0 - ; RPU-NEXT: Live-out: - ; RPU-NEXT: Live-thr: - ; RPU-NEXT: 0 0 - ; - ; RPD-LABEL: name: test_partially_used_early_clobber_def - ; RPD: Live-in: - ; RPD-NEXT: SGPR VGPR - ; RPD-NEXT: 0 0 - ; RPD-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3 - ; RPD-NEXT: 4 0 - ; RPD-NEXT: 8 0 early-clobber %1:sgpr_128 = COPY %0:sgpr_128 - ; RPD-NEXT: 1 0 - ; RPD-NEXT: 1 0 S_NOP 0, implicit %1.sub1:sgpr_128 - ; RPD-NEXT: 0 0 - ; RPD-NEXT: Live-out: - ; RPD-NEXT: Live-thr: - ; RPD-NEXT: 0 0 + ; RP-LABEL: name: test_partially_used_early_clobber_def + ; RP: Live-in: + ; RP-NEXT: SGPR VGPR + ; RP-NEXT: 0 0 + ; RP-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3 + ; RP-NEXT: 4 0 + ; RP-NEXT: 8 0 early-clobber %1:sgpr_128 = COPY %0:sgpr_128 + ; RP-NEXT: 1 0 + ; RP-NEXT: 1 0 S_NOP 0, implicit %1.sub1:sgpr_128 + ; RP-NEXT: 0 0 + ; RP-NEXT: Live-out: + ; RP-NEXT: Live-thr: + ; RP-NEXT: 0 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3 early-clobber %1:sgpr_128 = COPY %0:sgpr_128 S_NOP 0, implicit %1.sub1 @@ -752,7 +697,7 @@ body: | ; RPU-NEXT: 0 0 ; RPU-NEXT: 4 0 %0:sgpr_128 = COPY $sgpr0_sgpr1_sgpr2_sgpr3 ; RPU-NEXT: 4 0 - ; RPU-NEXT: 7 0 %1:sgpr_128 = COPY %0:sgpr_128, implicit-def %2:sgpr_128, implicit-def early-clobber %3:sgpr_128, implicit-def dead early-clobber %4:sgpr_128 + ; RPU-NEXT: 16 0 %1:sgpr_128 = COPY %0:sgpr_128, implicit-def %2:sgpr_128, implicit-def early-clobber %3:sgpr_128, implicit-def dead early-clobber %4:sgpr_128 ; RPU-NEXT: 6 0 ; RPU-NEXT: 6 0 S_NOP 0, implicit %1.sub1:sgpr_128, implicit %2.sub0_sub1:sgpr_128, implicit %3.sub0_sub1_sub2:sgpr_128 ; RPU-NEXT: 0 0 diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll index 81346996431f..b78f6412cd67 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-any.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900" ; ASM: amdhsa.target: amdgcn-amd-amdhsa--gfx900 ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x12C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_ANY_V4 (0x100) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -30,4 +37,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll index de518bb99370..a3c75e099753 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-not-supported.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx700" ; ASM: amdhsa.target: amdgcn-amd-amdhsa--gfx700 ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x22) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX700 (0x22) ; ELF-NEXT: ] @@ -29,4 +36,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll index b43d5941e8cc..d4d8af8c1a99 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-off.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack-" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack-' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x22C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -31,4 +38,4 @@ entry: attributes #0 = { "target-features"="-xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll index fc2bb4f4a280..9ca8b055a279 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-all-on.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack+" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack+' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x32C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -32,4 +39,4 @@ entry: attributes #0 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll index 27c58e400e13..fd3f5878469e 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-1.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack-" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack-' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x22C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -32,4 +39,4 @@ entry: attributes #0 = { "target-features"="-xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll index a97f8b2d9cd6..34673dd5b891 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-off-2.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack-" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack-' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x22C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -32,4 +39,4 @@ entry: attributes #0 = { "target-features"="-xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll index 9ef7d7acd3c1..c283ece7e8bd 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-1.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack+" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack+' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x32C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -32,4 +39,4 @@ entry: attributes #0 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll index 5d32346745f8..869254cae525 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-mul-func-xnack-any-on-2.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack+" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack+' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x32C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -32,4 +39,4 @@ entry: attributes #0 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll index d6778ec1f1fa..b1bcb34c8aee 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-any.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900" ; ASM: amdhsa.target: amdgcn-amd-amdhsa--gfx900 ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x12C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_ANY_V4 (0x100) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -20,4 +27,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll index efa09a61e630..cc04eb0e661d 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-not-supported.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx700 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx700" ; ASM: amdhsa.target: amdgcn-amd-amdhsa--gfx700 ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x22) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX700 (0x22) ; ELF-NEXT: ] @@ -19,4 +26,4 @@ entry: } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll index 630453656570..e84778b526f1 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-off.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack-" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack-' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x22C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_OFF_V4 (0x200) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -22,4 +29,4 @@ entry: attributes #0 = { "target-features"="-xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll index f99ceccec96a..a1ab6ed5f082 100644 --- a/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll +++ b/llvm/test/CodeGen/AMDGPU/tid-one-func-xnack-on.ll @@ -1,14 +1,21 @@ -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 < %s | FileCheck --check-prefixes=ASM %s -; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --filetype=obj < %s | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 | FileCheck --check-prefixes=ASM,ASM5 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s + +; RUN: sed 's/CODE_OBJECT_VERSION/500/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=4 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF4 %s +; RUN: sed 's/CODE_OBJECT_VERSION/400/g' %s | llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 --amdhsa-code-object-version=5 --filetype=obj | llvm-readobj --file-headers - | FileCheck --check-prefixes=ELF,ELF5 %s ; ASM: .amdgcn_target "amdgcn-amd-amdhsa--gfx900:xnack+" ; ASM: amdhsa.target: 'amdgcn-amd-amdhsa--gfx900:xnack+' ; ASM: amdhsa.version: ; ASM: - 1 -; ASM: - 1 +; ASM4: - 1 +; ASM5: - 2 ; ELF: OS/ABI: AMDGPU_HSA (0x40) -; ELF: ABIVersion: 2 +; ELF4: ABIVersion: 2 +; ELF5: ABIVersion: 3 ; ELF: Flags [ (0x32C) ; ELF-NEXT: EF_AMDGPU_FEATURE_XNACK_ON_V4 (0x300) ; ELF-NEXT: EF_AMDGPU_MACH_AMDGCN_GFX900 (0x2C) @@ -22,4 +29,4 @@ entry: attributes #0 = { "target-features"="+xnack" } !llvm.module.flags = !{!0} -!0 = !{i32 1, !"amdgpu_code_object_version", i32 400} +!0 = !{i32 1, !"amdgpu_code_object_version", i32 CODE_OBJECT_VERSION} diff --git a/llvm/test/CodeGen/ARM/aapcs-hfa-code.ll b/llvm/test/CodeGen/ARM/aapcs-hfa-code.ll index e32f19ef6745..dabbb1e38a86 100644 --- a/llvm/test/CodeGen/ARM/aapcs-hfa-code.ll +++ b/llvm/test/CodeGen/ARM/aapcs-hfa-code.ll @@ -104,10 +104,8 @@ define arm_aapcs_vfpcc void @test_1double_nosplit([4 x float], [4 x double], [3 ; CHECK-NEXT: push {r11, lr} ; CHECK-NEXT: .pad #8 ; CHECK-NEXT: sub sp, sp, #8 -; CHECK-NEXT: movw r1, #0 -; CHECK-NEXT: mov r0, #0 -; CHECK-NEXT: movt r1, #16368 -; CHECK-NEXT: strd r0, r1, [sp] +; CHECK-NEXT: vmov.f64 d16, #1.000000e+00 +; CHECK-NEXT: vstr d16, [sp] ; CHECK-NEXT: bl test_1double_nosplit ; CHECK-NEXT: add sp, sp, #8 ; CHECK-NEXT: pop {r11, pc} @@ -138,10 +136,8 @@ define arm_aapcs_vfpcc void @test_1double_misaligned([4 x double], [4 x double], ; CHECK-NEXT: push {r11, lr} ; CHECK-NEXT: .pad #16 ; CHECK-NEXT: sub sp, sp, #16 -; CHECK-NEXT: movw r1, #0 -; CHECK-NEXT: mov r0, #0 -; CHECK-NEXT: movt r1, #16368 -; CHECK-NEXT: strd r0, r1, [sp, #8] +; CHECK-NEXT: vmov.f64 d16, #1.000000e+00 +; CHECK-NEXT: vstr d16, [sp, #8] ; CHECK-NEXT: bl test_1double_misaligned ; CHECK-NEXT: add sp, sp, #16 ; CHECK-NEXT: pop {r11, pc} diff --git a/llvm/test/CodeGen/ARM/ha-alignstack-call.ll b/llvm/test/CodeGen/ARM/ha-alignstack-call.ll index e861fe397f84..7e2a911c8928 100644 --- a/llvm/test/CodeGen/ARM/ha-alignstack-call.ll +++ b/llvm/test/CodeGen/ARM/ha-alignstack-call.ll @@ -300,16 +300,16 @@ entry: ret double %call } ; CHECK-LABEL: g2_1_call: -; CHECK: movw r0, #0 -; CHECK: mov r1, #0 -; CHECK: movt r0, #16352 -; CHECK: str r1, [sp] -; CHECK: stmib sp, {r0, r1} -; CHECK: str r1, [sp, #12] -; CHECK: str r1, [sp, #16] -; CHECK: str r1, [sp, #20] -; CHECK: str r1, [sp, #24] -; CHECK: str r1, [sp, #28] +; CHECK: vmov.f64 d16, #5.000000e-01 +; CHECK: mov r0, #0 +; CHECK: str r0, [sp, #8] +; CHECK: str r0, [sp, #12] +; CHECK: str r0, [sp, #16] +; CHECK: vmov.i32 d0, #0x0 +; CHECK: str r0, [sp, #20] +; CHECK: str r0, [sp, #24] +; CHECK: str r0, [sp, #28] +; CHECK: vstr d16, [sp] ; CHECK: bl g2_1 ; pass in memory, alignment 8 diff --git a/llvm/test/CodeGen/ARM/vector-store.ll b/llvm/test/CodeGen/ARM/vector-store.ll index a8a1031637af..9c8ea7a2c440 100644 --- a/llvm/test/CodeGen/ARM/vector-store.ll +++ b/llvm/test/CodeGen/ARM/vector-store.ll @@ -403,17 +403,14 @@ define void @v3i8store(ptr %p) { ; CHECK-LABEL: v3i8store: ; CHECK: @ %bb.0: ; CHECK-NEXT: sub sp, #4 -; CHECK-NEXT: vmov.i32 d16, #0xff -; CHECK-NEXT: mov r1, sp -; CHECK-NEXT: vmov.i32 d17, #0x0 -; CHECK-NEXT: movs r2, #0 -; CHECK-NEXT: vand d16, d17, d16 -; CHECK-NEXT: vst1.32 {d16[0]}, [r1:32] -; CHECK-NEXT: vld1.32 {d16[0]}, [r1:32] +; CHECK-NEXT: movs r1, #0 +; CHECK-NEXT: mov r2, sp +; CHECK-NEXT: str r1, [sp] +; CHECK-NEXT: vld1.32 {d16[0]}, [r2:32] +; CHECK-NEXT: strb r1, [r0, #2] ; CHECK-NEXT: vmovl.u16 q8, d16 -; CHECK-NEXT: strb r2, [r0, #2] -; CHECK-NEXT: vmov.32 r1, d16[0] -; CHECK-NEXT: strh r1, [r0] +; CHECK-NEXT: vmov.32 r2, d16[0] +; CHECK-NEXT: strh r2, [r0] ; CHECK-NEXT: add sp, #4 ; CHECK-NEXT: bx lr store <3 x i8> zeroinitializer, ptr %p, align 4 diff --git a/llvm/test/CodeGen/BPF/GlobalISel/ir-translator-ret.ll b/llvm/test/CodeGen/BPF/GlobalISel/ir-translator-ret.ll new file mode 100644 index 000000000000..7a014f7841fc --- /dev/null +++ b/llvm/test/CodeGen/BPF/GlobalISel/ir-translator-ret.ll @@ -0,0 +1,7 @@ +; RUN: llc -mtriple=bpfel -global-isel -verify-machineinstrs -stop-after=irtranslator < %s | FileCheck %s + +; CHECK: name: f +; CHECK: RET +define void @f() { + ret void +} diff --git a/llvm/test/CodeGen/DirectX/UAVMetadata.ll b/llvm/test/CodeGen/DirectX/UAVMetadata.ll index e86d53cd7afc..3d95723d6e49 100644 --- a/llvm/test/CodeGen/DirectX/UAVMetadata.ll +++ b/llvm/test/CodeGen/DirectX/UAVMetadata.ll @@ -37,16 +37,16 @@ target triple = "dxil-pc-shadermodel6.0-library" !hlsl.uavs = !{!0, !1, !2, !3, !4, !5, !6, !7, !8, !9} -!0 = !{ptr @Zero, !"RWBuffer", i32 10, i32 0, i32 0} -!1 = !{ptr @One, !"Buffer>", i32 10, i32 1, i32 0} -!2 = !{ptr @Two, !"Buffer", i32 10, i32 2, i32 0} -!3 = !{ptr @Three, !"Buffer", i32 10, i32 3, i32 0} -!4 = !{ptr @Four, !"ByteAddressBuffer", i32 11, i32 5, i32 0} -!5 = !{ptr @Five, !"StructuredBuffer", i32 12, i32 6, i32 0} -!6 = !{ptr @Six, !"RasterizerOrderedBuffer", i32 10, i32 7, i32 0} -!7 = !{ptr @Seven, !"RasterizerOrderedStructuredBuffer", i32 12, i32 8, i32 0} -!8 = !{ptr @Eight, !"RasterizerOrderedByteAddressBuffer", i32 11, i32 9, i32 0} -!9 = !{ptr @Nine, !"RWBuffer", i32 10, i32 10, i32 2} +!0 = !{ptr @Zero, !"RWBuffer", i32 10, i1 false, i32 0, i32 0} +!1 = !{ptr @One, !"Buffer>", i32 10, i1 false, i32 1, i32 0} +!2 = !{ptr @Two, !"Buffer", i32 10, i1 false, i32 2, i32 0} +!3 = !{ptr @Three, !"Buffer", i32 10, i1 false, i32 3, i32 0} +!4 = !{ptr @Four, !"ByteAddressBuffer", i32 11, i1 false, i32 5, i32 0} +!5 = !{ptr @Five, !"StructuredBuffer", i32 12, i1 false, i32 6, i32 0} +!6 = !{ptr @Six, !"RasterizerOrderedBuffer", i32 10, i1 true, i32 7, i32 0} +!7 = !{ptr @Seven, !"RasterizerOrderedStructuredBuffer", i32 12, i1 true, i32 8, i32 0} +!8 = !{ptr @Eight, !"RasterizerOrderedByteAddressBuffer", i32 11, i1 true, i32 9, i32 0} +!9 = !{ptr @Nine, !"RWBuffer", i32 10, i1 false, i32 10, i32 2} ; CHECK: !dx.resources = !{[[ResList:[!][0-9]+]]} diff --git a/llvm/test/CodeGen/DirectX/cbuf.ll b/llvm/test/CodeGen/DirectX/cbuf.ll index 6640654f730e..d07cc1e880b1 100644 --- a/llvm/test/CodeGen/DirectX/cbuf.ll +++ b/llvm/test/CodeGen/DirectX/cbuf.ll @@ -34,4 +34,4 @@ attributes #1 = { nocallback nofree nosync nounwind readnone speculatable willre !hlsl.cbufs = !{!1} -!1 = !{ptr @A.cb., !"A.cb.ty", i32 13, i32 2, i32 1} +!1 = !{ptr @A.cb., !"A.cb.ty", i32 13, i1 false, i32 2, i32 1} diff --git a/llvm/test/CodeGen/DirectX/legacy_cb_layout_0.ll b/llvm/test/CodeGen/DirectX/legacy_cb_layout_0.ll index 50821467d5dd..0cfb839746b9 100644 --- a/llvm/test/CodeGen/DirectX/legacy_cb_layout_0.ll +++ b/llvm/test/CodeGen/DirectX/legacy_cb_layout_0.ll @@ -11,4 +11,4 @@ target triple = "dxil-unknown-shadermodel6.7-library" !hlsl.cbufs = !{!1} -!1 = !{ptr @A.cb., !"A.cb.ty", i32 13, i32 2, i32 0} +!1 = !{ptr @A.cb., !"A.cb.ty", i32 13, i1 false, i32 2, i32 0} diff --git a/llvm/test/CodeGen/DirectX/legacy_cb_layout_1.ll b/llvm/test/CodeGen/DirectX/legacy_cb_layout_1.ll index 8a38ccd8ef78..b6d29f8d18d7 100644 --- a/llvm/test/CodeGen/DirectX/legacy_cb_layout_1.ll +++ b/llvm/test/CodeGen/DirectX/legacy_cb_layout_1.ll @@ -34,4 +34,4 @@ target triple = "dxil-unknown-shadermodel6.7-library" !hlsl.cbufs = !{!0} -!0 = !{ptr @B.cb., !"B.cb.ty", i32 13, i32 1, i32 0} +!0 = !{ptr @B.cb., !"B.cb.ty", i32 13, i1 false, i32 1, i32 0} diff --git a/llvm/test/CodeGen/DirectX/legacy_cb_layout_2.ll b/llvm/test/CodeGen/DirectX/legacy_cb_layout_2.ll index b669538846ab..d023d7906fdc 100644 --- a/llvm/test/CodeGen/DirectX/legacy_cb_layout_2.ll +++ b/llvm/test/CodeGen/DirectX/legacy_cb_layout_2.ll @@ -47,5 +47,5 @@ target triple = "dxil-unknown-shadermodel6.7-library" !hlsl.cbufs = !{!0, !1} -!0 = !{ptr @B.cb., !"B.cb.ty", i32 13, i32 1, i32 0} -!1 = !{ptr @B.cb..1, !"B.cb.ty", i32 13, i32 2, i32 0} +!0 = !{ptr @B.cb., !"B.cb.ty", i32 13, i1 false, i32 1, i32 0} +!1 = !{ptr @B.cb..1, !"B.cb.ty", i32 13, i1 false, i32 2, i32 0} diff --git a/llvm/test/CodeGen/DirectX/legacy_cb_layout_3.ll b/llvm/test/CodeGen/DirectX/legacy_cb_layout_3.ll index afd46d08b5c7..38c2cd18b5ca 100644 --- a/llvm/test/CodeGen/DirectX/legacy_cb_layout_3.ll +++ b/llvm/test/CodeGen/DirectX/legacy_cb_layout_3.ll @@ -78,4 +78,4 @@ target triple = "dxil-unknown-shadermodel6.7-library" @D.cb. = external local_unnamed_addr constant { i32, %struct.B, half, %struct.C, double } !hlsl.cbufs = !{!0} -!0 = !{ptr @D.cb., !"D.cb.ty", i32 13, i32 1, i32 0} +!0 = !{ptr @D.cb., !"D.cb.ty", i32 13, i1 false, i32 1, i32 0} diff --git a/llvm/test/CodeGen/Generic/opt-codegen-no-target-machine.ll b/llvm/test/CodeGen/Generic/opt-codegen-no-target-machine.ll index 413f09ba55dd..b7f0e7d02a21 100644 --- a/llvm/test/CodeGen/Generic/opt-codegen-no-target-machine.ll +++ b/llvm/test/CodeGen/Generic/opt-codegen-no-target-machine.ll @@ -1,3 +1,3 @@ -; RUN: not --crash opt %s -dwarfehprepare -o - 2>&1 | FileCheck %s +; RUN: not --crash opt %s -dwarf-eh-prepare -o - 2>&1 | FileCheck %s ; CHECK: Trying to construct TargetPassConfig without a target machine. Scheduling a CodeGen pass without a target triple set? diff --git a/llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll b/llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll new file mode 100644 index 000000000000..af18c52b096c --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lasx/fma-v4f64.ll @@ -0,0 +1,804 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lasx --fp-contract=fast < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-FAST +; RUN: llc --mtriple=loongarch64 --mattr=+lasx --fp-contract=on < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-ON +; RUN: llc --mtriple=loongarch64 --mattr=+lasx --fp-contract=off < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-OFF + +define void @xvfmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfadd.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfadd.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul<4 x double> %v0, %v1 + %add = fadd<4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +define void @xvfmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul<4 x double> %v0, %v1 + %sub = fsub<4 x double> %mul, %v2 + store <4 x double> %sub, ptr %res + ret void +} + +define void @xvfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfadd.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfadd.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul<4 x double> %v0, %v1 + %add = fadd<4 x double> %mul, %v2 + %negadd = fneg<4 x double> %add + store <4 x double> %negadd, ptr %res + ret void +} + +define void @xvfnmadd_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmadd_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmadd_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr1, $xr1, 63 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmadd_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr1, $xr1, 63 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg nsz<4 x double> %v0 + %negv2 = fneg nsz<4 x double> %v2 + %mul = fmul nsz<4 x double> %negv0, %v1 + %add = fadd nsz<4 x double> %mul, %negv2 + store <4 x double> %add, ptr %res + ret void +} + +;; Check that xvfnmadd.d is not emitted. +define void @not_xvfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_xvfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_xvfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr1, $xr1, 63 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_xvfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr1, $xr1, 63 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg<4 x double> %v0 + %negv2 = fneg<4 x double> %v2 + %mul = fmul<4 x double> %negv0, %v1 + %add = fadd<4 x double> %mul, %negv2 + store <4 x double> %add, ptr %res + ret void +} + +define void @xvfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv2 = fneg<4 x double> %v2 + %mul = fmul<4 x double> %v0, %v1 + %add = fadd<4 x double> %mul, %negv2 + %neg = fneg<4 x double> %add + store <4 x double> %neg, ptr %res + ret void +} + +define void @xvfnmsub_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmsub_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmsub_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmsub_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg nsz<4 x double> %v0 + %mul = fmul nsz<4 x double> %negv0, %v1 + %add = fadd nsz<4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +;; Check that xvfnmsub.d is not emitted. +define void @not_xvfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_xvfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_xvfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_xvfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.d $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg<4 x double> %v0 + %mul = fmul<4 x double> %negv0, %v1 + %add = fadd<4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +define void @contract_xvfmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %add = fadd contract <4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +define void @contract_xvfmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %sub = fsub contract <4 x double> %mul, %v2 + store <4 x double> %sub, ptr %res + ret void +} + +define void @contract_xvfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %add = fadd contract <4 x double> %mul, %v2 + %negadd = fneg contract <4 x double> %add + store <4 x double> %negadd, ptr %res + ret void +} + +define void @contract_xvfnmadd_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmadd_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmadd_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmadd_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg contract nsz<4 x double> %v0 + %negv2 = fneg contract nsz<4 x double> %v2 + %mul = fmul contract nsz<4 x double> %negv0, %v1 + %add = fadd contract nsz<4 x double> %mul, %negv2 + store <4 x double> %add, ptr %res + ret void +} + +;; Check that xvfnmadd.d is not emitted. +define void @not_contract_xvfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_xvfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_xvfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-ON-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_xvfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-OFF-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg contract <4 x double> %v0 + %negv2 = fneg contract <4 x double> %v2 + %mul = fmul contract <4 x double> %negv0, %v1 + %add = fadd contract <4 x double> %mul, %negv2 + store <4 x double> %add, ptr %res + ret void +} + +define void @contract_xvfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv2 = fneg contract <4 x double> %v2 + %mul = fmul contract <4 x double> %v0, %v1 + %add = fadd contract <4 x double> %mul, %negv2 + %neg = fneg contract <4 x double> %add + store <4 x double> %neg, ptr %res + ret void +} + +define void @contract_xvfnmsub_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmsub_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmsub_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmsub_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg contract nsz<4 x double> %v0 + %mul = fmul contract nsz<4 x double> %negv0, %v1 + %add = fadd contract nsz<4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +;; Check that xvfnmsub.d is not emitted. +define void @not_contract_xvfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_xvfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_xvfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-ON-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_xvfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.d $xr2, $xr2, 63 +; CONTRACT-OFF-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %negv0 = fneg contract <4 x double> %v0 + %mul = fmul contract <4 x double> %negv0, %v1 + %add = fadd contract <4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +define void @xvfmadd_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmadd_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmadd_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmadd_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %add = fadd contract <4 x double> %mul, %v2 + store <4 x double> %add, ptr %res + ret void +} + +define void @xvfmsub_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmsub_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmsub_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmsub_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %sub = fsub contract <4 x double> %mul, %v2 + store <4 x double> %sub, ptr %res + ret void +} + +define void @xvfnmadd_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmadd_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmadd_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmadd_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %add = fadd contract <4 x double> %mul, %v2 + %negadd = fneg contract <4 x double> %add + store <4 x double> %negadd, ptr %res + ret void +} + +define void @xvfnmsub_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmsub_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmsub_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmsub_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.d $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %v2 = load <4 x double>, ptr %a2 + %mul = fmul contract <4 x double> %v0, %v1 + %negv2 = fneg contract <4 x double> %v2 + %add = fadd contract <4 x double> %negv2, %mul + %negadd = fneg contract <4 x double> %add + store <4 x double> %negadd, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll b/llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll new file mode 100644 index 000000000000..b7b3cb3a2e66 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lasx/fma-v8f32.ll @@ -0,0 +1,804 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lasx --fp-contract=fast < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-FAST +; RUN: llc --mtriple=loongarch64 --mattr=+lasx --fp-contract=on < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-ON +; RUN: llc --mtriple=loongarch64 --mattr=+lasx --fp-contract=off < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-OFF + +define void @xvfmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfadd.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfadd.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul<8 x float> %v0, %v1 + %add = fadd<8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +define void @xvfmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul<8 x float> %v0, %v1 + %sub = fsub<8 x float> %mul, %v2 + store <8 x float> %sub, ptr %res + ret void +} + +define void @xvfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfadd.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfadd.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul<8 x float> %v0, %v1 + %add = fadd<8 x float> %mul, %v2 + %negadd = fneg<8 x float> %add + store <8 x float> %negadd, ptr %res + ret void +} + +define void @xvfnmadd_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmadd_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmadd_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr1, $xr1, 31 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmadd_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr1, $xr1, 31 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg nsz<8 x float> %v0 + %negv2 = fneg nsz<8 x float> %v2 + %mul = fmul nsz<8 x float> %negv0, %v1 + %add = fadd nsz<8 x float> %mul, %negv2 + store <8 x float> %add, ptr %res + ret void +} + +;; Check that fnmadd.s is not emitted. +define void @not_xvfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_xvfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_xvfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr1, $xr1, 31 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_xvfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr1, $xr1, 31 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg<8 x float> %v0 + %negv2 = fneg<8 x float> %v2 + %mul = fmul<8 x float> %negv0, %v1 + %add = fadd<8 x float> %mul, %negv2 + store <8 x float> %add, ptr %res + ret void +} + +define void @xvfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr0, $xr1 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv2 = fneg<8 x float> %v2 + %mul = fmul<8 x float> %v0, %v1 + %add = fadd<8 x float> %mul, %negv2 + %neg = fneg<8 x float> %add + store <8 x float> %neg, ptr %res + ret void +} + +define void @xvfnmsub_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmsub_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmsub_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmsub_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg nsz<8 x float> %v0 + %mul = fmul nsz<8 x float> %negv0, %v1 + %add = fadd nsz<8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +;; Check that fnmsub.s is not emitted. +define void @not_xvfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_xvfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_xvfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-ON-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-ON-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_xvfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmul.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a3, 0 +; CONTRACT-OFF-NEXT: xvfsub.s $xr0, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg<8 x float> %v0 + %mul = fmul<8 x float> %negv0, %v1 + %add = fadd<8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +define void @contract_xvfmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %add = fadd contract <8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +define void @contract_xvfmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %sub = fsub contract <8 x float> %mul, %v2 + store <8 x float> %sub, ptr %res + ret void +} + +define void @contract_xvfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %add = fadd contract <8 x float> %mul, %v2 + %negadd = fneg contract <8 x float> %add + store <8 x float> %negadd, ptr %res + ret void +} + +define void @contract_xvfnmadd_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmadd_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmadd_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmadd_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg contract nsz<8 x float> %v0 + %negv2 = fneg contract nsz<8 x float> %v2 + %mul = fmul contract nsz<8 x float> %negv0, %v1 + %add = fadd contract nsz<8 x float> %mul, %negv2 + store <8 x float> %add, ptr %res + ret void +} + +;; Check that fnmadd.s is not emitted. +define void @not_contract_xvfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_xvfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_xvfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-ON-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_xvfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-OFF-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg contract <8 x float> %v0 + %negv2 = fneg contract <8 x float> %v2 + %mul = fmul contract <8 x float> %negv0, %v1 + %add = fadd contract <8 x float> %mul, %negv2 + store <8 x float> %add, ptr %res + ret void +} + +define void @contract_xvfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv2 = fneg contract <8 x float> %v2 + %mul = fmul contract <8 x float> %v0, %v1 + %add = fadd contract <8 x float> %mul, %negv2 + %neg = fneg contract <8 x float> %add + store <8 x float> %neg, ptr %res + ret void +} + +define void @contract_xvfnmsub_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_xvfnmsub_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_xvfnmsub_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_xvfnmsub_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg contract nsz<8 x float> %v0 + %mul = fmul contract nsz<8 x float> %negv0, %v1 + %add = fadd contract nsz<8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +;; Check that fnmsub.s is not emitted. +define void @not_contract_xvfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_xvfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_xvfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-ON-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_xvfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvbitrevi.w $xr2, $xr2, 31 +; CONTRACT-OFF-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %negv0 = fneg contract <8 x float> %v0 + %mul = fmul contract <8 x float> %negv0, %v1 + %add = fadd contract <8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +define void @xvfmadd_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmadd_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmadd_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmadd_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %add = fadd contract <8 x float> %mul, %v2 + store <8 x float> %add, ptr %res + ret void +} + +define void @xvfmsub_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfmsub_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfmsub_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfmsub_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %sub = fsub contract <8 x float> %mul, %v2 + store <8 x float> %sub, ptr %res + ret void +} + +define void @xvfnmadd_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmadd_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmadd_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmadd_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmadd.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %add = fadd contract <8 x float> %mul, %v2 + %negadd = fneg contract <8 x float> %add + store <8 x float> %negadd, ptr %res + ret void +} + +define void @xvfnmsub_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: xvfnmsub_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-FAST-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-FAST-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-FAST-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-FAST-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: xvfnmsub_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-ON-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-ON-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-ON-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-ON-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: xvfnmsub_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: xvld $xr0, $a3, 0 +; CONTRACT-OFF-NEXT: xvld $xr1, $a2, 0 +; CONTRACT-OFF-NEXT: xvld $xr2, $a1, 0 +; CONTRACT-OFF-NEXT: xvfnmsub.s $xr0, $xr2, $xr1, $xr0 +; CONTRACT-OFF-NEXT: xvst $xr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %v2 = load <8 x float>, ptr %a2 + %mul = fmul contract <8 x float> %v0, %v1 + %negv2 = fneg contract <8 x float> %v2 + %add = fadd contract <8 x float> %negv2, %mul + %negadd = fneg contract <8 x float> %add + store <8 x float> %negadd, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/fsqrt.ll b/llvm/test/CodeGen/LoongArch/lasx/fsqrt.ll new file mode 100644 index 000000000000..c4a881bdeae9 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lasx/fsqrt.ll @@ -0,0 +1,65 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s + +;; fsqrt +define void @sqrt_v8f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: sqrt_v8f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvfsqrt.s $xr0, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0, align 16 + %sqrt = call <8 x float> @llvm.sqrt.v8f32 (<8 x float> %v0) + store <8 x float> %sqrt, ptr %res, align 16 + ret void +} + +define void @sqrt_v4f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: sqrt_v4f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvfsqrt.d $xr0, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0, align 16 + %sqrt = call <4 x double> @llvm.sqrt.v4f64 (<4 x double> %v0) + store <4 x double> %sqrt, ptr %res, align 16 + ret void +} + +;; 1.0 / (fsqrt vec) +define void @one_div_sqrt_v8f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_div_sqrt_v8f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvfrsqrt.s $xr0, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0, align 16 + %sqrt = call <8 x float> @llvm.sqrt.v8f32 (<8 x float> %v0) + %div = fdiv <8 x float> , %sqrt + store <8 x float> %div, ptr %res, align 16 + ret void +} + +define void @one_div_sqrt_v4f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_div_sqrt_v4f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvfrsqrt.d $xr0, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0, align 16 + %sqrt = call <4 x double> @llvm.sqrt.v4f64 (<4 x double> %v0) + %div = fdiv <4 x double> , %sqrt + store <4 x double> %div, ptr %res, align 16 + ret void +} + +declare <8 x float> @llvm.sqrt.v8f32(<8 x float>) +declare <4 x double> @llvm.sqrt.v4f64(<4 x double>) diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fcmp.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fcmp.ll new file mode 100644 index 000000000000..ef67dbc100c0 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fcmp.ll @@ -0,0 +1,692 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s + +;; TREU +define void @v8f32_fcmp_true(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_true: +; CHECK: # %bb.0: +; CHECK-NEXT: xvrepli.b $xr0, -1 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp true <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +;; FALSE +define void @v4f64_fcmp_false(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_false: +; CHECK: # %bb.0: +; CHECK-NEXT: xvrepli.b $xr0, 0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp false <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETOEQ +define void @v8f32_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_oeq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.ceq.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp oeq <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_oeq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.ceq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp oeq <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETUEQ +define void @v8f32_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ueq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cueq.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ueq <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ueq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cueq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ueq <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETEQ +define void @v8f32_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.ceq.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp fast oeq <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.ceq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp fast ueq <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETOLE +define void @v8f32_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ole: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cle.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ole <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ole: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ole <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETULE +define void @v8f32_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cule.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ule <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cule.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ule <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETLE +define void @v8f32_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_le: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cle.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp fast ole <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_le: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp fast ule <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETOLT +define void @v8f32_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_olt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.clt.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp olt <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_olt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp olt <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETULT +define void @v8f32_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cult.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ult <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cult.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ult <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETLT +define void @v8f32_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_lt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.clt.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp fast olt <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_lt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp fast ult <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETONE +define void @v8f32_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_one: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cne.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp one <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_one: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cne.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp one <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETUNE +define void @v8f32_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_une: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cune.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp une <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_une: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cune.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp une <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETNE +define void @v8f32_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cne.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp fast one <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cne.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp fast une <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETO +define void @v8f32_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ord: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cor.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ord <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ord: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cor.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ord <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETUO +define void @v8f32_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_uno: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cun.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp uno <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_uno: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvfcmp.cun.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp uno <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETOGT +define void @v8f32_fcmp_ogt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ogt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.clt.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ogt <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ogt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ogt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ogt <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGT +define void @v8f32_fcmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cult.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp ugt <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cult.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp ugt <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGT +define void @v8f32_fcmp_gt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_gt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.clt.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp fast ogt <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_gt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_gt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.clt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp fast ugt <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETOGE +define void @v8f32_fcmp_oge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_oge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cle.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp oge <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_oge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_oge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp oge <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGE +define void @v8f32_fcmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cule.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp uge <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cule.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp uge <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGE +define void @v8f32_fcmp_ge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8f32_fcmp_ge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cle.s $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x float>, ptr %a0 + %v1 = load <8 x float>, ptr %a1 + %cmp = fcmp fast oge <8 x float> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4f64_fcmp_ge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f64_fcmp_ge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvfcmp.cle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x double>, ptr %a0 + %v1 = load <4 x double>, ptr %a1 + %cmp = fcmp fast uge <4 x double> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fdiv.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fdiv.ll index 284121a79a49..6004565b0b78 100644 --- a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fdiv.ll +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fdiv.ll @@ -32,3 +32,32 @@ entry: store <4 x double> %v2, ptr %res ret void } + +;; 1.0 / vec +define void @one_fdiv_v8f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_fdiv_v8f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvfrecip.s $xr0, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %div = fdiv <8 x float> , %v0 + store <8 x float> %div, ptr %res + ret void +} + +define void @one_fdiv_v4f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_fdiv_v4f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvfrecip.d $xr0, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %div = fdiv <4 x double> , %v0 + store <4 x double> %div, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fneg.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fneg.ll new file mode 100644 index 000000000000..5eb468fc55a0 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/fneg.ll @@ -0,0 +1,29 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s + +define void @fneg_v8f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: fneg_v8f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvbitrevi.w $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <8 x float>, ptr %a0 + %v1 = fneg <8 x float> %v0 + store <8 x float> %v1, ptr %res + ret void +} +define void @fneg_v4f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: fneg_v4f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvbitrevi.d $xr0, $xr0, 63 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x double>, ptr %a0 + %v1 = fneg <4 x double> %v0 + store <4 x double> %v1, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/icmp.ll b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/icmp.ll new file mode 100644 index 000000000000..6693fe0f6ec7 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lasx/ir-instruction/icmp.ll @@ -0,0 +1,939 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lasx < %s | FileCheck %s + +;; SETEQ +define void @v32i8_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v32i8_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvseqi.b $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %cmp = icmp eq <32 x i8> %v0, + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v32i8_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp eq <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i16_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvseqi.h $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %cmp = icmp eq <16 x i16> %v0, + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp eq <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i32_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvseqi.w $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %cmp = icmp eq <8 x i32> %v0, + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp eq <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i64_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvseqi.d $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %cmp = icmp eq <4 x i64> %v0, + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp eq <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETLE +define void @v32i8_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v32i8_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.b $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %cmp = icmp sle <32 x i8> %v0, + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v32i8_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp sle <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i16_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.h $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %cmp = icmp sle <16 x i16> %v0, + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp sle <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i32_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.w $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %cmp = icmp sle <8 x i32> %v0, + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp sle <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i64_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.d $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %cmp = icmp sle <4 x i64> %v0, + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp sle <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETULE +define void @v32i8_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v32i8_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.bu $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %cmp = icmp ule <32 x i8> %v0, + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v32i8_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.bu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp ule <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i16_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.hu $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %cmp = icmp ule <16 x i16> %v0, + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.hu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp ule <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i32_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.wu $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %cmp = icmp ule <8 x i32> %v0, + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.wu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp ule <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i64_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslei.du $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %cmp = icmp ule <4 x i64> %v0, + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvsle.du $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp ule <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETLT +define void @v32i8_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v32i8_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.b $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %cmp = icmp slt <32 x i8> %v0, + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v32i8_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp slt <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i16_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.h $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %cmp = icmp slt <16 x i16> %v0, + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp slt <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i32_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.w $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %cmp = icmp slt <8 x i32> %v0, + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp slt <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i64_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.d $xr0, $xr0, 15 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %cmp = icmp slt <4 x i64> %v0, + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp slt <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; SETULT +define void @v32i8_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v32i8_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.bu $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %cmp = icmp ult <32 x i8> %v0, + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v32i8_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.bu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp ult <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i16_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.hu $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %cmp = icmp ult <16 x i16> %v0, + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.hu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp ult <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i32_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.wu $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %cmp = icmp ult <8 x i32> %v0, + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.wu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp ult <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i64_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvslti.du $xr0, $xr0, 31 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %cmp = icmp ult <4 x i64> %v0, + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvslt.du $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp ult <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETNE +define void @v32i8_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvxori.b $xr0, $xr0, 255 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp ne <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvrepli.b $xr1, -1 +; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp ne <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvrepli.b $xr1, -1 +; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp ne <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a2, 0 +; CHECK-NEXT: xvld $xr1, $a1, 0 +; CHECK-NEXT: xvseq.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvrepli.b $xr1, -1 +; CHECK-NEXT: xvxor.v $xr0, $xr0, $xr1 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp ne <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGE +define void @v32i8_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp sge <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp sge <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp sge <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp sge <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGE +define void @v32i8_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.bu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp uge <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.hu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp uge <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.wu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp uge <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvsle.du $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp uge <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGT +define void @v32i8_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.b $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp sgt <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.h $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp sgt <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.w $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp sgt <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.d $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp sgt <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGT +define void @v32i8_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v32i8_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.bu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <32 x i8>, ptr %a0 + %v1 = load <32 x i8>, ptr %a1 + %cmp = icmp ugt <32 x i8> %v0, %v1 + %ext = sext <32 x i1> %cmp to <32 x i8> + store <32 x i8> %ext, ptr %res + ret void +} + +define void @v16i16_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i16_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.hu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i16>, ptr %a0 + %v1 = load <16 x i16>, ptr %a1 + %cmp = icmp ugt <16 x i16> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i16> + store <16 x i16> %ext, ptr %res + ret void +} + +define void @v8i32_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i32_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.wu $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i32>, ptr %a0 + %v1 = load <8 x i32>, ptr %a1 + %cmp = icmp ugt <8 x i32> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i32> + store <8 x i32> %ext, ptr %res + ret void +} + +define void @v4i64_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i64_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: xvld $xr0, $a1, 0 +; CHECK-NEXT: xvld $xr1, $a2, 0 +; CHECK-NEXT: xvslt.du $xr0, $xr1, $xr0 +; CHECK-NEXT: xvst $xr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i64>, ptr %a0 + %v1 = load <4 x i64>, ptr %a1 + %cmp = icmp ugt <4 x i64> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i64> + store <4 x i64> %ext, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll b/llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll new file mode 100644 index 000000000000..8e0459b4afab --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lsx/fma-v2f64.ll @@ -0,0 +1,804 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lsx --fp-contract=fast < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-FAST +; RUN: llc --mtriple=loongarch64 --mattr=+lsx --fp-contract=on < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-ON +; RUN: llc --mtriple=loongarch64 --mattr=+lsx --fp-contract=off < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-OFF + +define void @vfmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfadd.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfadd.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul<2 x double> %v0, %v1 + %add = fadd<2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +define void @vfmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul<2 x double> %v0, %v1 + %sub = fsub<2 x double> %mul, %v2 + store <2 x double> %sub, ptr %res + ret void +} + +define void @vfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfadd.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfadd.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul<2 x double> %v0, %v1 + %add = fadd<2 x double> %mul, %v2 + %negadd = fneg<2 x double> %add + store <2 x double> %negadd, ptr %res + ret void +} + +define void @vfnmadd_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmadd_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmadd_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.d $vr1, $vr1, 63 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmadd_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr1, $vr1, 63 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg nsz<2 x double> %v0 + %negv2 = fneg nsz<2 x double> %v2 + %mul = fmul nsz<2 x double> %negv0, %v1 + %add = fadd nsz<2 x double> %mul, %negv2 + store <2 x double> %add, ptr %res + ret void +} + +;; Check that vfnmadd.d is not emitted. +define void @not_vfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_vfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_vfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.d $vr1, $vr1, 63 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_vfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr1, $vr1, 63 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg<2 x double> %v0 + %negv2 = fneg<2 x double> %v2 + %mul = fmul<2 x double> %negv0, %v1 + %add = fadd<2 x double> %mul, %negv2 + store <2 x double> %add, ptr %res + ret void +} + +define void @vfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv2 = fneg<2 x double> %v2 + %mul = fmul<2 x double> %v0, %v1 + %add = fadd<2 x double> %mul, %negv2 + %neg = fneg<2 x double> %add + store <2 x double> %neg, ptr %res + ret void +} + +define void @vfnmsub_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmsub_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmsub_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmsub_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg nsz<2 x double> %v0 + %mul = fmul nsz<2 x double> %negv0, %v1 + %add = fadd nsz<2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +;; Check that vfnmsub.d is not emitted. +define void @not_vfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_vfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_vfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_vfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.d $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg<2 x double> %v0 + %mul = fmul<2 x double> %negv0, %v1 + %add = fadd<2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +define void @contract_vfmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %add = fadd contract <2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +define void @contract_vfmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %sub = fsub contract <2 x double> %mul, %v2 + store <2 x double> %sub, ptr %res + ret void +} + +define void @contract_vfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %add = fadd contract <2 x double> %mul, %v2 + %negadd = fneg contract <2 x double> %add + store <2 x double> %negadd, ptr %res + ret void +} + +define void @contract_vfnmadd_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmadd_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmadd_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmadd_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg contract nsz<2 x double> %v0 + %negv2 = fneg contract nsz<2 x double> %v2 + %mul = fmul contract nsz<2 x double> %negv0, %v1 + %add = fadd contract nsz<2 x double> %mul, %negv2 + store <2 x double> %add, ptr %res + ret void +} + +;; Check that vfnmadd.d is not emitted. +define void @not_contract_vfnmadd_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_vfnmadd_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_vfnmadd_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-ON-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_vfnmadd_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-OFF-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg contract <2 x double> %v0 + %negv2 = fneg contract <2 x double> %v2 + %mul = fmul contract <2 x double> %negv0, %v1 + %add = fadd contract <2 x double> %mul, %negv2 + store <2 x double> %add, ptr %res + ret void +} + +define void @contract_vfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv2 = fneg contract <2 x double> %v2 + %mul = fmul contract <2 x double> %v0, %v1 + %add = fadd contract <2 x double> %mul, %negv2 + %neg = fneg contract <2 x double> %add + store <2 x double> %neg, ptr %res + ret void +} + +define void @contract_vfnmsub_d_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmsub_d_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmsub_d_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmsub_d_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg contract nsz<2 x double> %v0 + %mul = fmul contract nsz<2 x double> %negv0, %v1 + %add = fadd contract nsz<2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +;; Check that vfnmsub.d is not emitted. +define void @not_contract_vfnmsub_d(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_vfnmsub_d: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_vfnmsub_d: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-ON-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_vfnmsub_d: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.d $vr2, $vr2, 63 +; CONTRACT-OFF-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %negv0 = fneg contract <2 x double> %v0 + %mul = fmul contract <2 x double> %negv0, %v1 + %add = fadd contract <2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +define void @vfmadd_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmadd_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmadd_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmadd_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %add = fadd contract <2 x double> %mul, %v2 + store <2 x double> %add, ptr %res + ret void +} + +define void @vfmsub_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmsub_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmsub_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmsub_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %sub = fsub contract <2 x double> %mul, %v2 + store <2 x double> %sub, ptr %res + ret void +} + +define void @vfnmadd_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmadd_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmadd_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmadd_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmadd.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %add = fadd contract <2 x double> %mul, %v2 + %negadd = fneg contract <2 x double> %add + store <2 x double> %negadd, ptr %res + ret void +} + +define void @vfnmsub_d_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmsub_d_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmsub_d_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmsub_d_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmsub.d $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %v2 = load <2 x double>, ptr %a2 + %mul = fmul contract <2 x double> %v0, %v1 + %negv2 = fneg contract <2 x double> %v2 + %add = fadd contract <2 x double> %negv2, %mul + %negadd = fneg contract <2 x double> %add + store <2 x double> %negadd, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll b/llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll new file mode 100644 index 000000000000..7efbd61c0c4f --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lsx/fma-v4f32.ll @@ -0,0 +1,804 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lsx --fp-contract=fast < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-FAST +; RUN: llc --mtriple=loongarch64 --mattr=+lsx --fp-contract=on < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-ON +; RUN: llc --mtriple=loongarch64 --mattr=+lsx --fp-contract=off < %s \ +; RUN: | FileCheck %s --check-prefix=CONTRACT-OFF + +define void @vfmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfadd.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfadd.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul<4 x float> %v0, %v1 + %add = fadd<4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +define void @vfmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul<4 x float> %v0, %v1 + %sub = fsub<4 x float> %mul, %v2 + store <4 x float> %sub, ptr %res + ret void +} + +define void @vfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfadd.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfadd.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul<4 x float> %v0, %v1 + %add = fadd<4 x float> %mul, %v2 + %negadd = fneg<4 x float> %add + store <4 x float> %negadd, ptr %res + ret void +} + +define void @vfnmadd_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmadd_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmadd_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.w $vr1, $vr1, 31 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmadd_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr1, $vr1, 31 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg nsz<4 x float> %v0 + %negv2 = fneg nsz<4 x float> %v2 + %mul = fmul nsz<4 x float> %negv0, %v1 + %add = fadd nsz<4 x float> %mul, %negv2 + store <4 x float> %add, ptr %res + ret void +} + +;; Check that vfnmadd.s is not emitted. +define void @not_vfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_vfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_vfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.w $vr1, $vr1, 31 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_vfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr1, $vr1, 31 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg<4 x float> %v0 + %negv2 = fneg<4 x float> %v2 + %mul = fmul<4 x float> %negv0, %v1 + %add = fadd<4 x float> %mul, %negv2 + store <4 x float> %add, ptr %res + ret void +} + +define void @vfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-ON-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr0, $vr1 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv2 = fneg<4 x float> %v2 + %mul = fmul<4 x float> %v0, %v1 + %add = fadd<4 x float> %mul, %negv2 + %neg = fneg<4 x float> %add + store <4 x float> %neg, ptr %res + ret void +} + +define void @vfnmsub_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmsub_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmsub_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmsub_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg nsz<4 x float> %v0 + %mul = fmul nsz<4 x float> %negv0, %v1 + %add = fadd nsz<4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +;; Check that vfnmsub.s is not emitted. +define void @not_vfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_vfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_vfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a1, 0 +; CONTRACT-ON-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vld $vr1, $a3, 0 +; CONTRACT-ON-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_vfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a1, 0 +; CONTRACT-OFF-NEXT: vfmul.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vld $vr1, $a3, 0 +; CONTRACT-OFF-NEXT: vfsub.s $vr0, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg<4 x float> %v0 + %mul = fmul<4 x float> %negv0, %v1 + %add = fadd<4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +define void @contract_vfmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %add = fadd contract <4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +define void @contract_vfmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %sub = fsub contract <4 x float> %mul, %v2 + store <4 x float> %sub, ptr %res + ret void +} + +define void @contract_vfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %add = fadd contract <4 x float> %mul, %v2 + %negadd = fneg contract <4 x float> %add + store <4 x float> %negadd, ptr %res + ret void +} + +define void @contract_vfnmadd_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmadd_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmadd_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmadd_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg contract nsz<4 x float> %v0 + %negv2 = fneg contract nsz<4 x float> %v2 + %mul = fmul contract nsz<4 x float> %negv0, %v1 + %add = fadd contract nsz<4 x float> %mul, %negv2 + store <4 x float> %add, ptr %res + ret void +} + +;; Check that vfnmadd.s is not emitted. +define void @not_contract_vfnmadd_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_vfnmadd_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_vfnmadd_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-ON-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_vfnmadd_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-OFF-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg contract <4 x float> %v0 + %negv2 = fneg contract <4 x float> %v2 + %mul = fmul contract <4 x float> %negv0, %v1 + %add = fadd contract <4 x float> %mul, %negv2 + store <4 x float> %add, ptr %res + ret void +} + +define void @contract_vfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv2 = fneg contract <4 x float> %v2 + %mul = fmul contract <4 x float> %v0, %v1 + %add = fadd contract <4 x float> %mul, %negv2 + %neg = fneg contract <4 x float> %add + store <4 x float> %neg, ptr %res + ret void +} + +define void @contract_vfnmsub_s_nsz(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: contract_vfnmsub_s_nsz: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: contract_vfnmsub_s_nsz: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: contract_vfnmsub_s_nsz: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg contract nsz<4 x float> %v0 + %mul = fmul contract nsz<4 x float> %negv0, %v1 + %add = fadd contract nsz<4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +;; Check that vfnmsub.s is not emitted. +define void @not_contract_vfnmsub_s(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: not_contract_vfnmsub_s: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: not_contract_vfnmsub_s: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-ON-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: not_contract_vfnmsub_s: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vbitrevi.w $vr2, $vr2, 31 +; CONTRACT-OFF-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %negv0 = fneg contract <4 x float> %v0 + %mul = fmul contract <4 x float> %negv0, %v1 + %add = fadd contract <4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +define void @vfmadd_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmadd_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmadd_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmadd_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %add = fadd contract <4 x float> %mul, %v2 + store <4 x float> %add, ptr %res + ret void +} + +define void @vfmsub_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfmsub_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfmsub_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfmsub_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %sub = fsub contract <4 x float> %mul, %v2 + store <4 x float> %sub, ptr %res + ret void +} + +define void @vfnmadd_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmadd_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmadd_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmadd_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmadd.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %add = fadd contract <4 x float> %mul, %v2 + %negadd = fneg contract <4 x float> %add + store <4 x float> %negadd, ptr %res + ret void +} + +define void @vfnmsub_s_contract(ptr %res, ptr %a0, ptr %a1, ptr %a2) nounwind { +; CONTRACT-FAST-LABEL: vfnmsub_s_contract: +; CONTRACT-FAST: # %bb.0: # %entry +; CONTRACT-FAST-NEXT: vld $vr0, $a3, 0 +; CONTRACT-FAST-NEXT: vld $vr1, $a2, 0 +; CONTRACT-FAST-NEXT: vld $vr2, $a1, 0 +; CONTRACT-FAST-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-FAST-NEXT: vst $vr0, $a0, 0 +; CONTRACT-FAST-NEXT: ret +; +; CONTRACT-ON-LABEL: vfnmsub_s_contract: +; CONTRACT-ON: # %bb.0: # %entry +; CONTRACT-ON-NEXT: vld $vr0, $a3, 0 +; CONTRACT-ON-NEXT: vld $vr1, $a2, 0 +; CONTRACT-ON-NEXT: vld $vr2, $a1, 0 +; CONTRACT-ON-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-ON-NEXT: vst $vr0, $a0, 0 +; CONTRACT-ON-NEXT: ret +; +; CONTRACT-OFF-LABEL: vfnmsub_s_contract: +; CONTRACT-OFF: # %bb.0: # %entry +; CONTRACT-OFF-NEXT: vld $vr0, $a3, 0 +; CONTRACT-OFF-NEXT: vld $vr1, $a2, 0 +; CONTRACT-OFF-NEXT: vld $vr2, $a1, 0 +; CONTRACT-OFF-NEXT: vfnmsub.s $vr0, $vr2, $vr1, $vr0 +; CONTRACT-OFF-NEXT: vst $vr0, $a0, 0 +; CONTRACT-OFF-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %v2 = load <4 x float>, ptr %a2 + %mul = fmul contract <4 x float> %v0, %v1 + %negv2 = fneg contract <4 x float> %v2 + %add = fadd contract <4 x float> %negv2, %mul + %negadd = fneg contract <4 x float> %add + store <4 x float> %negadd, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lsx/fsqrt.ll b/llvm/test/CodeGen/LoongArch/lsx/fsqrt.ll new file mode 100644 index 000000000000..a57bc1ca0e94 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lsx/fsqrt.ll @@ -0,0 +1,65 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s + +;; fsqrt +define void @sqrt_v4f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: sqrt_v4f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vfsqrt.s $vr0, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0, align 16 + %sqrt = call <4 x float> @llvm.sqrt.v4f32 (<4 x float> %v0) + store <4 x float> %sqrt, ptr %res, align 16 + ret void +} + +define void @sqrt_v2f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: sqrt_v2f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vfsqrt.d $vr0, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0, align 16 + %sqrt = call <2 x double> @llvm.sqrt.v2f64 (<2 x double> %v0) + store <2 x double> %sqrt, ptr %res, align 16 + ret void +} + +;; 1.0 / (fsqrt vec) +define void @one_div_sqrt_v4f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_div_sqrt_v4f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vfrsqrt.s $vr0, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0, align 16 + %sqrt = call <4 x float> @llvm.sqrt.v4f32 (<4 x float> %v0) + %div = fdiv <4 x float> , %sqrt + store <4 x float> %div, ptr %res, align 16 + ret void +} + +define void @one_div_sqrt_v2f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_div_sqrt_v2f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vfrsqrt.d $vr0, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0, align 16 + %sqrt = call <2 x double> @llvm.sqrt.v2f64 (<2 x double> %v0) + %div = fdiv <2 x double> , %sqrt + store <2 x double> %div, ptr %res, align 16 + ret void +} + +declare <4 x float> @llvm.sqrt.v4f32(<4 x float>) +declare <2 x double> @llvm.sqrt.v2f64(<2 x double>) diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fcmp.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fcmp.ll new file mode 100644 index 000000000000..53fbf0b2f86f --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fcmp.ll @@ -0,0 +1,692 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s + +;; TREU +define void @v4f32_fcmp_true(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_true: +; CHECK: # %bb.0: +; CHECK-NEXT: vrepli.b $vr0, -1 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp true <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +;; FALSE +define void @v2f64_fcmp_false(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_false: +; CHECK: # %bb.0: +; CHECK-NEXT: vrepli.b $vr0, 0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp false <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETOEQ +define void @v4f32_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_oeq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.ceq.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp oeq <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_oeq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_oeq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.ceq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp oeq <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETUEQ +define void @v4f32_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ueq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cueq.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ueq <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ueq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ueq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cueq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ueq <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETEQ +define void @v4f32_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.ceq.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp fast oeq <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.ceq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp fast ueq <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETOLE +define void @v4f32_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ole: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cle.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ole <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ole(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ole: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ole <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETULE +define void @v4f32_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cule.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ule <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cule.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ule <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETLE +define void @v4f32_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_le: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cle.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp fast ole <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_le(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_le: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp fast ule <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETOLT +define void @v4f32_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_olt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.clt.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp olt <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_olt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_olt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.clt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp olt <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETULT +define void @v4f32_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cult.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ult <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cult.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ult <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETLT +define void @v4f32_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_lt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.clt.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp fast olt <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_lt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_lt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.clt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp fast ult <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETONE +define void @v4f32_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_one: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cne.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp one <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_one(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_one: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cne.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp one <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETUNE +define void @v4f32_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_une: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cune.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp une <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_une(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_une: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cune.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp une <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETNE +define void @v4f32_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cne.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp fast one <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cne.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp fast une <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETO +define void @v4f32_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ord: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cor.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ord <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ord(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ord: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cor.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ord <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETUO +define void @v4f32_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_uno: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cun.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp uno <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_uno(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_uno: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vfcmp.cun.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp uno <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETOGT +define void @v4f32_fcmp_ogt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ogt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.clt.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ogt <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ogt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ogt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.clt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ogt <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGT +define void @v4f32_fcmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cult.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp ugt <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cult.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp ugt <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGT +define void @v4f32_fcmp_gt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_gt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.clt.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp fast ogt <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_gt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_gt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.clt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp fast ugt <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETOGE +define void @v4f32_fcmp_oge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_oge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cle.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp oge <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_oge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_oge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp oge <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGE +define void @v4f32_fcmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cule.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp uge <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cule.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp uge <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGE +define void @v4f32_fcmp_ge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4f32_fcmp_ge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cle.s $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x float>, ptr %a0 + %v1 = load <4 x float>, ptr %a1 + %cmp = fcmp fast oge <4 x float> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2f64_fcmp_ge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2f64_fcmp_ge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vfcmp.cle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x double>, ptr %a0 + %v1 = load <2 x double>, ptr %a1 + %cmp = fcmp fast uge <2 x double> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fdiv.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fdiv.ll index eb7c8bd9616e..5f1ee9e4d212 100644 --- a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fdiv.ll +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fdiv.ll @@ -32,3 +32,32 @@ entry: store <2 x double> %v2, ptr %res ret void } + +;; 1.0 / vec +define void @one_fdiv_v4f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_fdiv_v4f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vfrecip.s $vr0, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %div = fdiv <4 x float> , %v0 + store <4 x float> %div, ptr %res + ret void +} + +define void @one_fdiv_v2f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: one_fdiv_v2f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vfrecip.d $vr0, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %div = fdiv <2 x double> , %v0 + store <2 x double> %div, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fneg.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fneg.ll new file mode 100644 index 000000000000..795c1ac8b368 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/fneg.ll @@ -0,0 +1,29 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s + +define void @fneg_v4f32(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: fneg_v4f32: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vbitrevi.w $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <4 x float>, ptr %a0 + %v1 = fneg <4 x float> %v0 + store <4 x float> %v1, ptr %res + ret void +} +define void @fneg_v2f64(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: fneg_v2f64: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vbitrevi.d $vr0, $vr0, 63 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret +entry: + %v0 = load <2 x double>, ptr %a0 + %v1 = fneg <2 x double> %v0 + store <2 x double> %v1, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/icmp.ll b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/icmp.ll new file mode 100644 index 000000000000..448f3fa6c6e0 --- /dev/null +++ b/llvm/test/CodeGen/LoongArch/lsx/ir-instruction/icmp.ll @@ -0,0 +1,939 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py +; RUN: llc --mtriple=loongarch64 --mattr=+lsx < %s | FileCheck %s + +;; SETEQ +define void @v16i8_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i8_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vseqi.b $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %cmp = icmp eq <16 x i8> %v0, + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v16i8_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp eq <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i16_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vseqi.h $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %cmp = icmp eq <8 x i16> %v0, + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp eq <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i32_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vseqi.w $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %cmp = icmp eq <4 x i32> %v0, + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp eq <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_eq_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v2i64_icmp_eq_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vseqi.d $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %cmp = icmp eq <2 x i64> %v0, + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_eq(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_eq: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp eq <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETLE +define void @v16i8_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i8_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.b $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %cmp = icmp sle <16 x i8> %v0, + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v16i8_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp sle <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i16_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.h $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %cmp = icmp sle <8 x i16> %v0, + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp sle <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i32_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.w $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %cmp = icmp sle <4 x i32> %v0, + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp sle <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_sle_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v2i64_icmp_sle_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.d $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %cmp = icmp sle <2 x i64> %v0, + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_sle(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_sle: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp sle <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETULE +define void @v16i8_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i8_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.bu $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %cmp = icmp ule <16 x i8> %v0, + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v16i8_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.bu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp ule <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i16_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.hu $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %cmp = icmp ule <8 x i16> %v0, + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.hu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp ule <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i32_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.wu $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %cmp = icmp ule <4 x i32> %v0, + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.wu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp ule <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_ule_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v2i64_icmp_ule_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslei.du $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %cmp = icmp ule <2 x i64> %v0, + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_ule(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_ule: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vsle.du $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp ule <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETLT +define void @v16i8_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i8_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.b $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %cmp = icmp slt <16 x i8> %v0, + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v16i8_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp slt <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i16_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.h $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %cmp = icmp slt <8 x i16> %v0, + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp slt <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i32_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.w $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %cmp = icmp slt <4 x i32> %v0, + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp slt <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_slt_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v2i64_icmp_slt_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.d $vr0, $vr0, 15 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %cmp = icmp slt <2 x i64> %v0, + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_slt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_slt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp slt <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; SETULT +define void @v16i8_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v16i8_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.bu $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %cmp = icmp ult <16 x i8> %v0, + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v16i8_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.bu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp ult <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v8i16_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.hu $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %cmp = icmp ult <8 x i16> %v0, + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.hu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp ult <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v4i32_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.wu $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %cmp = icmp ult <4 x i32> %v0, + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.wu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp ult <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_ult_imm(ptr %res, ptr %a0) nounwind { +; CHECK-LABEL: v2i64_icmp_ult_imm: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vslti.du $vr0, $vr0, 31 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %cmp = icmp ult <2 x i64> %v0, + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_ult(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_ult: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vslt.du $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp ult <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETNE +define void @v16i8_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vxori.b $vr0, $vr0, 255 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp ne <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vrepli.b $vr1, -1 +; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp ne <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vrepli.b $vr1, -1 +; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp ne <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_ne(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_ne: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a2, 0 +; CHECK-NEXT: vld $vr1, $a1, 0 +; CHECK-NEXT: vseq.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vrepli.b $vr1, -1 +; CHECK-NEXT: vxor.v $vr0, $vr0, $vr1 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp ne <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGE +define void @v16i8_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp sge <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp sge <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp sge <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_sge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_sge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp sge <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGE +define void @v16i8_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.bu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp uge <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.hu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp uge <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.wu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp uge <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_uge(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_uge: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vsle.du $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp uge <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETGT +define void @v16i8_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.b $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp sgt <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.h $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp sgt <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.w $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp sgt <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_sgt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_sgt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.d $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp sgt <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} + +;; Expand SETUGT +define void @v16i8_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v16i8_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.bu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <16 x i8>, ptr %a0 + %v1 = load <16 x i8>, ptr %a1 + %cmp = icmp ugt <16 x i8> %v0, %v1 + %ext = sext <16 x i1> %cmp to <16 x i8> + store <16 x i8> %ext, ptr %res + ret void +} + +define void @v8i16_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v8i16_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.hu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <8 x i16>, ptr %a0 + %v1 = load <8 x i16>, ptr %a1 + %cmp = icmp ugt <8 x i16> %v0, %v1 + %ext = sext <8 x i1> %cmp to <8 x i16> + store <8 x i16> %ext, ptr %res + ret void +} + +define void @v4i32_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v4i32_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.wu $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <4 x i32>, ptr %a0 + %v1 = load <4 x i32>, ptr %a1 + %cmp = icmp ugt <4 x i32> %v0, %v1 + %ext = sext <4 x i1> %cmp to <4 x i32> + store <4 x i32> %ext, ptr %res + ret void +} + +define void @v2i64_icmp_ugt(ptr %res, ptr %a0, ptr %a1) nounwind { +; CHECK-LABEL: v2i64_icmp_ugt: +; CHECK: # %bb.0: +; CHECK-NEXT: vld $vr0, $a1, 0 +; CHECK-NEXT: vld $vr1, $a2, 0 +; CHECK-NEXT: vslt.du $vr0, $vr1, $vr0 +; CHECK-NEXT: vst $vr0, $a0, 0 +; CHECK-NEXT: ret + %v0 = load <2 x i64>, ptr %a0 + %v1 = load <2 x i64>, ptr %a1 + %cmp = icmp ugt <2 x i64> %v0, %v1 + %ext = sext <2 x i1> %cmp to <2 x i64> + store <2 x i64> %ext, ptr %res + ret void +} diff --git a/llvm/test/CodeGen/Mips/compactbranches/beqc-bnec-register-constraint.ll b/llvm/test/CodeGen/Mips/compactbranches/beqc-bnec-register-constraint.ll index 4c8674d46e29..13e30676aa6f 100644 --- a/llvm/test/CodeGen/Mips/compactbranches/beqc-bnec-register-constraint.ll +++ b/llvm/test/CodeGen/Mips/compactbranches/beqc-bnec-register-constraint.ll @@ -1,5 +1,5 @@ -; RUN: llc -march=mips -mcpu=mips32r6 -O1 -start-after=dwarfehprepare < %s | FileCheck %s -; RUN: llc -march=mips64 -mcpu=mips64r6 -O1 -start-after=dwarfehprepare < %s | FileCheck %s +; RUN: llc -march=mips -mcpu=mips32r6 -O1 -start-after=dwarf-eh-prepare < %s | FileCheck %s +; RUN: llc -march=mips64 -mcpu=mips64r6 -O1 -start-after=dwarf-eh-prepare < %s | FileCheck %s ; beqc/bnec have the constraint that $rs < $rt && $rs != 0 && $rt != 0 diff --git a/llvm/test/CodeGen/Mips/pr49200.ll b/llvm/test/CodeGen/Mips/pr49200.ll index 80a2bdd4e95e..2a9f207b29e5 100644 --- a/llvm/test/CodeGen/Mips/pr49200.ll +++ b/llvm/test/CodeGen/Mips/pr49200.ll @@ -11,11 +11,10 @@ define dso_local void @foo() #0 { ; CHECK-LABEL: foo: ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: addiusp -24 -; CHECK-NEXT: li16 $2, 0 -; CHECK-NEXT: sw $2, 4($sp) -; CHECK-NEXT: sw $2, 0($sp) -; CHECK-NEXT: sw $2, 12($sp) -; CHECK-NEXT: sw $2, 8($sp) +; CHECK-NEXT: mtc1 $zero, $f0 +; CHECK-NEXT: mthc1 $zero, $f0 +; CHECK-NEXT: sdc1 $f0, 0($sp) +; CHECK-NEXT: sdc1 $f0, 8($sp) ; CHECK-NEXT: ldc1 $f0, 0($sp) ; CHECK-NEXT: sdc1 $f0, 16($sp) ; CHECK-NEXT: addiusp 24 diff --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll index 5a6ab2926b40..a9faa130d637 100644 --- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll +++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll @@ -227,3 +227,106 @@ define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 { %res = fpext <8 x bfloat> %load to <8 x float> ret <8 x float> %res } + +; CHECK-LABEL: test_fptosi_i16( +; CHECK: ld.param.b16 [[A:%rs[0-9]+]], [test_fptosi_i16_param_0]; +; SM80: cvt.f32.bf16 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rzi.s16.f32 [[C:%rs[0-9]+]], [[B]]; +; SM80: cvt.u32.u16 [[R:%r[0-9]+]], [[C]]; +; SM90: cvt.rzi.s16.bf16 [[B:%rs[0-9]+]], [[A]]; +; SM90: cvt.u32.u16 [[R:%r[0-9]+]], [[B]]; +; CHECK: st.param.b32 [func_retval0+0], [[R]]; +; CHECK: ret; +define i16 @test_fptosi_i16(bfloat %a) { + %r = fptosi bfloat %a to i16 + ret i16 %r +} + +; CHECK-LABEL: test_fptoui_i16( +; CHECK: ld.param.b16 [[A:%rs[0-9]+]], [test_fptoui_i16_param_0]; +; SM80: cvt.f32.bf16 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rzi.u16.f32 [[C:%rs[0-9]+]], [[B]]; +; SM80: cvt.u32.u16 [[R:%r[0-9]+]], [[C]]; +; SM90: cvt.rzi.u16.bf16 [[B:%rs[0-9]+]], [[A]]; +; SM90: cvt.u32.u16 [[R:%r[0-9]+]], [[B]]; +; CHECK: st.param.b32 [func_retval0+0], [[R]]; +; CHECK: ret; +define i16 @test_fptoui_i16(bfloat %a) { + %r = fptoui bfloat %a to i16 + ret i16 %r +} + +; CHECK-LABEL: test_sitofp_i16( +; CHECK: ld.param.u16 [[A:%rs[0-9]+]], [test_sitofp_i16_param_0]; +; SM80: cvt.rn.f32.s16 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; +; SM90: cvt.rn.bf16.s16 [[R:%rs[0-9]+]], [[A]]; +; CHECK: st.param.b16 [func_retval0+0], [[R]]; +; CHECK: ret; +define bfloat @test_sitofp_i16(i16 %a) { + %r = sitofp i16 %a to bfloat + ret bfloat %r +} + +; CHECK-LABEL: test_uitofp_i8( +; CHECK: ld.param.u8 %rs1, [test_uitofp_i8_param_0]; +; SM80: cvt.rn.f32.u16 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; +; SM90: cvt.rn.bf16.u16 [[R:%rs[0-9]+]], [[A]]; +; CHECK: st.param.b16 [func_retval0+0], [[R]]; +; CHECK: ret; +define bfloat @test_uitofp_i8(i8 %a) { + %r = uitofp i8 %a to bfloat + ret bfloat %r +} + +; CHECK-LABEL: test_uitofp_i1( +; CHECK: ld.param.u8 [[A:%rs[0-9]+]], [test_uitofp_i1_param_0]; +; CHECK: and.b16 [[B:%rs[0-9]+]], [[A]], 1; +; CHECK: setp.eq.b16 [[C:%p[0-9]+]], [[B]], 1; +; CHECK: selp.u32 [[D:%r[0-9]+]], 1, 0, [[C]]; +; SM80: cvt.rn.f32.u32 [[E:%f[0-9]+]], [[D]]; +; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[E]]; +; SM90: cvt.rn.bf16.u32 [[R:%rs[0-9]+]], [[D]]; +; CHECK: st.param.b16 [func_retval0+0], [[R]]; +; CHECK: ret; +define bfloat @test_uitofp_i1(i1 %a) { + %r = uitofp i1 %a to bfloat + ret bfloat %r +} + +; CHECK-LABEL: test_uitofp_i16( +; CHECK: ld.param.u16 [[A:%rs[0-9]+]], [test_uitofp_i16_param_0]; +; SM80: cvt.rn.f32.u16 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; +; SM90: cvt.rn.bf16.u16 [[R:%rs[0-9]+]], [[A]]; +; CHECK: st.param.b16 [func_retval0+0], [[R]]; +; CHECK: ret; +define bfloat @test_uitofp_i16(i16 %a) { + %r = uitofp i16 %a to bfloat + ret bfloat %r +} + +; CHECK-LABEL: test_uitofp_i32( +; CHECK: ld.param.u32 [[A:%r[0-9]+]], [test_uitofp_i32_param_0]; +; SM80: cvt.rn.f32.u32 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; +; SM90: cvt.rn.bf16.u32 [[R:%rs[0-9]+]], [[A]]; +; CHECK: st.param.b16 [func_retval0+0], [[R]]; +; CHECK: ret; +define bfloat @test_uitofp_i32(i32 %a) { + %r = uitofp i32 %a to bfloat + ret bfloat %r +} + +; CHECK-LABEL: test_uitofp_i64( +; CHECK: ld.param.u64 [[A:%rd[0-9]+]], [test_uitofp_i64_param_0]; +; SM80: cvt.rn.f32.u64 [[B:%f[0-9]+]], [[A]]; +; SM80: cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]]; +; SM90: cvt.rn.bf16.u64 [[R:%rs[0-9]+]], [[A]]; +; CHECK: st.param.b16 [func_retval0+0], [[R]]; +; CHECK: ret; +define bfloat @test_uitofp_i64(i64 %a) { + %r = uitofp i64 %a to bfloat + ret bfloat %r +} diff --git a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll index 7fba1aadcd98..c08f16c2644d 100644 --- a/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll +++ b/llvm/test/CodeGen/NVPTX/bf16x2-instructions.ll @@ -392,6 +392,16 @@ define <2 x bfloat> @test_fabs(<2 x bfloat> %a) #0 { ret <2 x bfloat> %r } +; CHECK-LABEL: test_fabs_add( +; CHECK: abs.bf16x2 +; CHECK: ret; +define <2 x bfloat> @test_fabs_add(<2 x bfloat> %a, <2 x bfloat> %b) #0 { + %s = fadd <2 x bfloat> %a, %a + %r = call <2 x bfloat> @llvm.fabs.f16(<2 x bfloat> %s) + %d = fadd <2 x bfloat> %r, %b + ret <2 x bfloat> %d +} + ; CHECK-LABEL: test_minnum( ; CHECK-DAG: ld.param.b32 [[AF0:%r[0-9]+]], [test_minnum_param_0]; diff --git a/llvm/test/CodeGen/PowerPC/aix-ehinfo-sym.ll b/llvm/test/CodeGen/PowerPC/aix-ehinfo-sym.ll index dfbc871cbdb9..06faf692cdca 100644 --- a/llvm/test/CodeGen/PowerPC/aix-ehinfo-sym.ll +++ b/llvm/test/CodeGen/PowerPC/aix-ehinfo-sym.ll @@ -4,6 +4,10 @@ ; RUN: llc -mtriple powerpc64-ibm-aix -fast-isel -verify-machineinstrs < %s | \ ; RUN: FileCheck %s +; RUN: llc -mtriple powerpc64-ibm-aix -verify-machineinstrs -filetype=obj \ +; RUN: -o %t.o < %s +; RUN: llvm-readobj --syms %t.o | FileCheck --check-prefix=SYM %s + ; Function Attrs: nounwind declare i32 @func1() #0 @@ -47,4 +51,11 @@ attributes #0 = { nounwind } attributes #1 = { mustprogress noinline optnone } ; CHECK: __ehinfo.0: -; CHECK: .tc __ehinfo.0[TC],__ehinfo.0 +; CHECK: .tc __ehinfo.0[TE],__ehinfo.0 + +; SYM: Symbol { +; SYM: Name: __ehinfo.0 +; SYM: CSECT Auxiliary Entry { +; SYM: StorageMappingClass: XMC_TE (0x16) +; SYM: } +; SYM: } diff --git a/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable-clobber-register.ll b/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable-clobber-register.ll index 0cfe120f0ae4..42bd4782c868 100644 --- a/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable-clobber-register.ll +++ b/llvm/test/CodeGen/PowerPC/aix-emit-tracebacktable-clobber-register.ll @@ -104,7 +104,7 @@ entry: ; COMMON-NEXT: # -- End function ; COMMON: .toc ; COMMON: L..C2: -; COMMON-NEXT: .tc __ehinfo.1[TC],__ehinfo.1 +; COMMON-NEXT: .tc __ehinfo.1[TE],__ehinfo.1 ; OBJ-DIS: 9c: 00 00 00 00 # Traceback table start diff --git a/llvm/test/CodeGen/PowerPC/aix-exception.ll b/llvm/test/CodeGen/PowerPC/aix-exception.ll index eabf29b97d01..5035d8e323d1 100644 --- a/llvm/test/CodeGen/PowerPC/aix-exception.ll +++ b/llvm/test/CodeGen/PowerPC/aix-exception.ll @@ -180,7 +180,7 @@ eh.resume: ; preds = %catch.dispatch ; ASM: L..C0: ; ASM: .tc _ZTIi[TC],_ZTIi[UA] ; ASM: L..C1: -; ASM: .tc __ehinfo.1[TC],__ehinfo.1 +; ASM: .tc __ehinfo.1[TE],__ehinfo.1 declare ptr @__cxa_allocate_exception(i32) declare void @__cxa_throw(ptr, ptr, ptr) diff --git a/llvm/test/CodeGen/PowerPC/aix-tls-gd-double.ll b/llvm/test/CodeGen/PowerPC/aix-tls-gd-double.ll index 46ce3bfd450b..c0ffb8154c69 100644 --- a/llvm/test/CodeGen/PowerPC/aix-tls-gd-double.ll +++ b/llvm/test/CodeGen/PowerPC/aix-tls-gd-double.ll @@ -24,7 +24,7 @@ define void @storesTGUninit(double %Val) #0 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C0(2) # target-flags(ppc-lo) @TGUninit +; SMALL32-NEXT: lwz 3, L..C0(2) # target-flags(ppc-tlsgdm) @TGUninit ; SMALL32-NEXT: lwz 4, L..C1(2) # target-flags(ppc-tlsgd) @TGUninit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -54,7 +54,7 @@ define void @storesTGUninit(double %Val) #0 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C0(2) # target-flags(ppc-lo) @TGUninit +; SMALL64-NEXT: ld 3, L..C0(2) # target-flags(ppc-tlsgdm) @TGUninit ; SMALL64-NEXT: ld 4, L..C1(2) # target-flags(ppc-tlsgd) @TGUninit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -90,7 +90,7 @@ define void @storesTGInit(double %Val) #0 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C2(2) # target-flags(ppc-lo) @TGInit +; SMALL32-NEXT: lwz 3, L..C2(2) # target-flags(ppc-tlsgdm) @TGInit ; SMALL32-NEXT: lwz 4, L..C3(2) # target-flags(ppc-tlsgd) @TGInit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -120,7 +120,7 @@ define void @storesTGInit(double %Val) #0 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C2(2) # target-flags(ppc-lo) @TGInit +; SMALL64-NEXT: ld 3, L..C2(2) # target-flags(ppc-tlsgdm) @TGInit ; SMALL64-NEXT: ld 4, L..C3(2) # target-flags(ppc-tlsgd) @TGInit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -156,7 +156,7 @@ define void @storesTIInit(double %Val) #0 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C4(2) # target-flags(ppc-lo) @TIInit +; SMALL32-NEXT: lwz 3, L..C4(2) # target-flags(ppc-tlsgdm) @TIInit ; SMALL32-NEXT: lwz 4, L..C5(2) # target-flags(ppc-tlsgd) @TIInit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -186,7 +186,7 @@ define void @storesTIInit(double %Val) #0 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C4(2) # target-flags(ppc-lo) @TIInit +; SMALL64-NEXT: ld 3, L..C4(2) # target-flags(ppc-tlsgdm) @TIInit ; SMALL64-NEXT: ld 4, L..C5(2) # target-flags(ppc-tlsgd) @TIInit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -222,7 +222,7 @@ define void @storesTWInit(double %Val) #0 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C6(2) # target-flags(ppc-lo) @TWInit +; SMALL32-NEXT: lwz 3, L..C6(2) # target-flags(ppc-tlsgdm) @TWInit ; SMALL32-NEXT: lwz 4, L..C7(2) # target-flags(ppc-tlsgd) @TWInit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -252,7 +252,7 @@ define void @storesTWInit(double %Val) #0 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C6(2) # target-flags(ppc-lo) @TWInit +; SMALL64-NEXT: ld 3, L..C6(2) # target-flags(ppc-tlsgdm) @TWInit ; SMALL64-NEXT: ld 4, L..C7(2) # target-flags(ppc-tlsgd) @TWInit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -288,7 +288,7 @@ define double @loadsTGUninit() #1 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C0(2) # target-flags(ppc-lo) @TGUninit +; SMALL32-NEXT: lwz 3, L..C0(2) # target-flags(ppc-tlsgdm) @TGUninit ; SMALL32-NEXT: lwz 4, L..C1(2) # target-flags(ppc-tlsgd) @TGUninit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -325,7 +325,7 @@ define double @loadsTGUninit() #1 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C0(2) # target-flags(ppc-lo) @TGUninit +; SMALL64-NEXT: ld 3, L..C0(2) # target-flags(ppc-tlsgdm) @TGUninit ; SMALL64-NEXT: ld 4, L..C1(2) # target-flags(ppc-tlsgd) @TGUninit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -370,7 +370,7 @@ define double @loadsTGInit() #1 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C2(2) # target-flags(ppc-lo) @TGInit +; SMALL32-NEXT: lwz 3, L..C2(2) # target-flags(ppc-tlsgdm) @TGInit ; SMALL32-NEXT: lwz 4, L..C3(2) # target-flags(ppc-tlsgd) @TGInit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -407,7 +407,7 @@ define double @loadsTGInit() #1 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C2(2) # target-flags(ppc-lo) @TGInit +; SMALL64-NEXT: ld 3, L..C2(2) # target-flags(ppc-tlsgdm) @TGInit ; SMALL64-NEXT: ld 4, L..C3(2) # target-flags(ppc-tlsgd) @TGInit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -452,7 +452,7 @@ define double @loadsTIInit() #1 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C4(2) # target-flags(ppc-lo) @TIInit +; SMALL32-NEXT: lwz 3, L..C4(2) # target-flags(ppc-tlsgdm) @TIInit ; SMALL32-NEXT: lwz 4, L..C5(2) # target-flags(ppc-tlsgd) @TIInit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -489,7 +489,7 @@ define double @loadsTIInit() #1 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C4(2) # target-flags(ppc-lo) @TIInit +; SMALL64-NEXT: ld 3, L..C4(2) # target-flags(ppc-tlsgdm) @TIInit ; SMALL64-NEXT: ld 4, L..C5(2) # target-flags(ppc-tlsgd) @TIInit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] @@ -534,7 +534,7 @@ define double @loadsTWInit() #1 { ; SMALL32: # %bb.0: # %entry ; SMALL32-NEXT: mflr 0 ; SMALL32-NEXT: stwu 1, -32(1) -; SMALL32-NEXT: lwz 3, L..C6(2) # target-flags(ppc-lo) @TWInit +; SMALL32-NEXT: lwz 3, L..C6(2) # target-flags(ppc-tlsgdm) @TWInit ; SMALL32-NEXT: lwz 4, L..C7(2) # target-flags(ppc-tlsgd) @TWInit ; SMALL32-NEXT: stw 0, 40(1) ; SMALL32-NEXT: bla .__tls_get_addr[PR] @@ -571,7 +571,7 @@ define double @loadsTWInit() #1 { ; SMALL64: # %bb.0: # %entry ; SMALL64-NEXT: mflr 0 ; SMALL64-NEXT: stdu 1, -48(1) -; SMALL64-NEXT: ld 3, L..C6(2) # target-flags(ppc-lo) @TWInit +; SMALL64-NEXT: ld 3, L..C6(2) # target-flags(ppc-tlsgdm) @TWInit ; SMALL64-NEXT: ld 4, L..C7(2) # target-flags(ppc-tlsgd) @TWInit ; SMALL64-NEXT: std 0, 64(1) ; SMALL64-NEXT: bla .__tls_get_addr[PR] diff --git a/llvm/test/CodeGen/PowerPC/aix-tls-gd-target-flags.ll b/llvm/test/CodeGen/PowerPC/aix-tls-gd-target-flags.ll index f52e213fee58..f55932c3088d 100644 --- a/llvm/test/CodeGen/PowerPC/aix-tls-gd-target-flags.ll +++ b/llvm/test/CodeGen/PowerPC/aix-tls-gd-target-flags.ll @@ -8,11 +8,11 @@ define signext i32 @foo() { ; CHECK-LABEL: name: foo ; CHECK: bb.0.entry: - ; CHECK-NEXT: [[LDtoc:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-lo) @a, $x2 :: (load (s64) from got) + ; CHECK-NEXT: [[LDtoc:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-tlsgdm) @a, $x2 :: (load (s64) from got) ; CHECK-NEXT: [[LDtoc1:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-tlsgd) @a, $x2 :: (load (s64) from got) ; CHECK-NEXT: [[TLSGDAIX8_:%[0-9]+]]:g8rc_and_g8rc_nox0 = TLSGDAIX8 killed [[LDtoc1]], killed [[LDtoc]] ; CHECK-NEXT: [[LWZ:%[0-9]+]]:gprc = LWZ 0, killed [[TLSGDAIX8_]] :: (dereferenceable load (s32) from @a) - ; CHECK-NEXT: [[LDtoc2:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-lo) @b, $x2 :: (load (s64) from got) + ; CHECK-NEXT: [[LDtoc2:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-tlsgdm) @b, $x2 :: (load (s64) from got) ; CHECK-NEXT: [[LDtoc3:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-tlsgd) @b, $x2 :: (load (s64) from got) ; CHECK-NEXT: [[TLSGDAIX8_1:%[0-9]+]]:g8rc_and_g8rc_nox0 = TLSGDAIX8 killed [[LDtoc3]], killed [[LDtoc2]] ; CHECK-NEXT: [[LWZ1:%[0-9]+]]:gprc = LWZ 0, killed [[TLSGDAIX8_1]] :: (dereferenceable load (s32) from @b) diff --git a/llvm/test/CodeGen/PowerPC/ctrloops-pseudo.ll b/llvm/test/CodeGen/PowerPC/ctrloops-pseudo.ll index 2fcc67087468..e7c49c9dcc7d 100644 --- a/llvm/test/CodeGen/PowerPC/ctrloops-pseudo.ll +++ b/llvm/test/CodeGen/PowerPC/ctrloops-pseudo.ll @@ -375,7 +375,7 @@ define i32 @test4(i32 %inp) { ; AIX64-NEXT: B %bb.2 ; AIX64-NEXT: {{ $}} ; AIX64-NEXT: bb.2.return: - ; AIX64-NEXT: [[LDtoc:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-lo) @tls_var, $x2 :: (load (s64) from got) + ; AIX64-NEXT: [[LDtoc:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-tlsgdm) @tls_var, $x2 :: (load (s64) from got) ; AIX64-NEXT: [[LDtoc1:%[0-9]+]]:g8rc = LDtoc target-flags(ppc-tlsgd) @tls_var, $x2 :: (load (s64) from got) ; AIX64-NEXT: [[TLSGDAIX8_:%[0-9]+]]:g8rc = TLSGDAIX8 killed [[LDtoc1]], killed [[LDtoc]] ; AIX64-NEXT: [[COPY2:%[0-9]+]]:gprc = COPY [[TLSGDAIX8_]].sub_32 @@ -405,7 +405,7 @@ define i32 @test4(i32 %inp) { ; AIX32-NEXT: B %bb.2 ; AIX32-NEXT: {{ $}} ; AIX32-NEXT: bb.2.return: - ; AIX32-NEXT: [[LWZtoc:%[0-9]+]]:gprc = LWZtoc target-flags(ppc-lo) @tls_var, $r2 :: (load (s32) from got) + ; AIX32-NEXT: [[LWZtoc:%[0-9]+]]:gprc = LWZtoc target-flags(ppc-tlsgdm) @tls_var, $r2 :: (load (s32) from got) ; AIX32-NEXT: [[LWZtoc1:%[0-9]+]]:gprc = LWZtoc target-flags(ppc-tlsgd) @tls_var, $r2 :: (load (s32) from got) ; AIX32-NEXT: [[TLSGDAIX:%[0-9]+]]:gprc = TLSGDAIX killed [[LWZtoc1]], killed [[LWZtoc]] ; AIX32-NEXT: [[ADD4_:%[0-9]+]]:gprc = ADD4 killed [[TLSGDAIX]], [[ISEL]] diff --git a/llvm/test/CodeGen/PowerPC/pcrel-tls-local-dynamic.ll b/llvm/test/CodeGen/PowerPC/pcrel-tls-local-dynamic.ll index 0b0fdbcaf0a4..da9844348f60 100644 --- a/llvm/test/CodeGen/PowerPC/pcrel-tls-local-dynamic.ll +++ b/llvm/test/CodeGen/PowerPC/pcrel-tls-local-dynamic.ll @@ -13,7 +13,7 @@ define nonnull ptr @LocalDynamicAddressLoad() { ; CHECK-S-LABEL: LocalDynamicAddressLoad: ; CHECK-S: paddi r3, 0, x@got@tlsld@pcrel, 1 ; CHECK-S-NEXT: bl __tls_get_addr@notoc(x@tlsld) - ; CHECK-S-NEXT: paddi r3, r3, x@DTPREL, 0 + ; CHECK-S-NEXT: paddi r3, r3, x@DTPREL ; CHECK-S-NEXT: addi r1, r1, 32 ; CHECK-S-NEXT: ld r0, 16(r1) ; CHECK-S-NEXT: mtlr r0 @@ -34,7 +34,7 @@ define i32 @LocalDynamicValueLoad() { ; CHECK-S-LABEL: LocalDynamicValueLoad: ; CHECK-S: paddi r3, 0, x@got@tlsld@pcrel, 1 ; CHECK-S-NEXT: bl __tls_get_addr@notoc(x@tlsld) - ; CHECK-S-NEXT: paddi r3, r3, x@DTPREL, 0 + ; CHECK-S-NEXT: paddi r3, r3, x@DTPREL ; CHECK-S-NEXT: lwz r3, 0(r3) ; CHECK-S-NEXT: addi r1, r1, 32 ; CHECK-S-NEXT: ld r0, 16(r1) diff --git a/llvm/test/CodeGen/PowerPC/pcrel-tls-local-exec.ll b/llvm/test/CodeGen/PowerPC/pcrel-tls-local-exec.ll index 77d18e1b64de..e4a22e0e2427 100644 --- a/llvm/test/CodeGen/PowerPC/pcrel-tls-local-exec.ll +++ b/llvm/test/CodeGen/PowerPC/pcrel-tls-local-exec.ll @@ -14,10 +14,10 @@ define dso_local ptr @LocalExecAddressLoad() { ; CHECK-S-LABEL: LocalExecAddressLoad: ; CHECK-S: # %bb.0: # %entry -; CHECK-S-NEXT: paddi r3, r13, x@TPREL, 0 +; CHECK-S-NEXT: paddi r3, r13, x@TPREL ; CHECK-S-NEXT: blr ; CHECK-O-LABEL: : -; CHECK-O: 0: paddi 3, 13, 0, 0 +; CHECK-O: 0: paddi 3, 13, 0 ; CHECK-O-NEXT: 0000000000000000: R_PPC64_TPREL34 x ; CHECK-O-NEXT: 8: blr entry: @@ -27,11 +27,11 @@ entry: define dso_local i32 @LocalExecValueLoad() { ; CHECK-S-LABEL: LocalExecValueLoad: ; CHECK-S: # %bb.0: # %entry -; CHECK-S-NEXT: paddi r3, r13, x@TPREL, 0 +; CHECK-S-NEXT: paddi r3, r13, x@TPREL ; CHECK-S-NEXT: lwz r3, 0(r3) ; CHECK-S-NEXT: blr ; CHECK-O-LABEL: : -; CHECK-O: 20: paddi 3, 13, 0, 0 +; CHECK-O: 20: paddi 3, 13, 0 ; CHECK-O-NEXT: 0000000000000020: R_PPC64_TPREL34 x ; CHECK-O-NEXT: 28: lwz 3, 0(3) ; CHECK-O-NEXT: 2c: blr @@ -43,11 +43,11 @@ entry: define dso_local void @LocalExecValueStore(i32 %in) { ; CHECK-S-LABEL: LocalExecValueStore: ; CHECK-S: # %bb.0: # %entry -; CHECK-S-NEXT: paddi r4, r13, x@TPREL, 0 +; CHECK-S-NEXT: paddi r4, r13, x@TPREL ; CHECK-S-NEXT: stw r3, 0(r4) ; CHECK-S-NEXT: blr ; CHECK-O-LABEL: : -; CHECK-O: 40: paddi 4, 13, 0, 0 +; CHECK-O: 40: paddi 4, 13, 0 ; CHECK-O-NEXT: 0000000000000040: R_PPC64_TPREL34 x ; CHECK-O-NEXT: 48: stw 3, 0(4) ; CHECK-O-NEXT: 4c: blr @@ -59,11 +59,11 @@ entry: define dso_local i32 @LocalExecValueLoadOffset() { ; CHECK-S-LABEL: LocalExecValueLoadOffset: ; CHECK-S: # %bb.0: # %entry -; CHECK-S-NEXT: paddi r3, r13, y@TPREL, 0 +; CHECK-S-NEXT: paddi r3, r13, y@TPREL ; CHECK-S-NEXT: lwz r3, 12(r3) ; CHECK-S-NEXT: blr ; CHECK-O-LABEL: : -; CHECK-O: 60: paddi 3, 13, 0, 0 +; CHECK-O: 60: paddi 3, 13, 0 ; CHECK-O-NEXT: 0000000000000060: R_PPC64_TPREL34 y ; CHECK-O-NEXT: 68: lwz 3, 12(3) ; CHECK-O-NEXT: 6c: blr @@ -76,11 +76,11 @@ entry: define dso_local ptr @LocalExecValueLoadOffsetNoLoad() { ; CHECK-S-LABEL: LocalExecValueLoadOffsetNoLoad: ; CHECK-S: # %bb.0: # %entry -; CHECK-S-NEXT: paddi r3, r13, y@TPREL, 0 +; CHECK-S-NEXT: paddi r3, r13, y@TPREL ; CHECK-S-NEXT: addi r3, r3, 12 ; CHECK-S-NEXT: blr ; CHECK-O-LABEL: : -; CHECK-O: 80: paddi 3, 13, 0, 0 +; CHECK-O: 80: paddi 3, 13, 0 ; CHECK-O-NEXT: 0000000000000080: R_PPC64_TPREL34 y ; CHECK-O-NEXT: 88: addi 3, 3, 12 ; CHECK-O-NEXT: 8c: blr diff --git a/llvm/test/CodeGen/PowerPC/pcrel-tls_get_addr_clobbers.ll b/llvm/test/CodeGen/PowerPC/pcrel-tls_get_addr_clobbers.ll index 38f55dac1ea8..905d13814ab7 100644 --- a/llvm/test/CodeGen/PowerPC/pcrel-tls_get_addr_clobbers.ll +++ b/llvm/test/CodeGen/PowerPC/pcrel-tls_get_addr_clobbers.ll @@ -13,7 +13,7 @@ define void @test(ptr %arg) { ; CHECK-NEXT: mr r30, r3 ; CHECK-NEXT: paddi r3, 0, x@got@tlsld@pcrel, 1 ; CHECK-NEXT: bl __tls_get_addr@notoc(x@tlsld) -; CHECK-NEXT: paddi r3, r3, x@DTPREL, 0 +; CHECK-NEXT: paddi r3, r3, x@DTPREL ; CHECK-NEXT: std r30, 0(r3) ; CHECK-NEXT: addi r1, r1, 48 ; CHECK-NEXT: ld r0, 16(r1) diff --git a/llvm/test/CodeGen/PowerPC/tls-crash.mir b/llvm/test/CodeGen/PowerPC/tls-crash.mir index 186ba6da325a..aa08790b36c9 100644 --- a/llvm/test/CodeGen/PowerPC/tls-crash.mir +++ b/llvm/test/CodeGen/PowerPC/tls-crash.mir @@ -18,8 +18,8 @@ body: | liveins: $x3 %0:g8rc = COPY killed $x3 - %1:g8rc_and_g8rc_nox0 = PADDI8pc 0, target-flags(ppc-pcrel, ppc-got, ppc-tlsld) @x - %2:g8rc_and_g8rc_nox0 = PADDIdtprel killed %1, target-flags(ppc-pcrel, ppc-got, ppc-tlsld) @x + %1:g8rc_and_g8rc_nox0 = PADDI8pc 0, target-flags(ppc-got-tlsld-pcrel) @x + %2:g8rc_and_g8rc_nox0 = PADDIdtprel killed %1, target-flags(ppc-got-tlsld-pcrel) @x STD killed %0, 0, killed %2 :: (store (s64) into `i8** bitcast (%0** @x to i8**)`) BLR8 implicit $lr8, implicit $rm ... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vacopy.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vacopy.ll new file mode 100644 index 000000000000..48d72108335e --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vacopy.ll @@ -0,0 +1,28 @@ +; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 4 +; RUN: llc -mtriple=riscv32 -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefix=RV32I %s +; RUN: llc -mtriple=riscv64 -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \ +; RUN: | FileCheck -check-prefix=RV64I %s + +declare void @llvm.va_copy(ptr, ptr) +define void @test_va_copy(ptr %dest_list, ptr %src_list) { + ; RV32I-LABEL: name: test_va_copy + ; RV32I: bb.1 (%ir-block.0): + ; RV32I-NEXT: liveins: $x10, $x11 + ; RV32I-NEXT: {{ $}} + ; RV32I-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32I-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x11 + ; RV32I-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[COPY]](p0), [[COPY1]](p0) + ; RV32I-NEXT: PseudoRET + ; + ; RV64I-LABEL: name: test_va_copy + ; RV64I: bb.1 (%ir-block.0): + ; RV64I-NEXT: liveins: $x10, $x11 + ; RV64I-NEXT: {{ $}} + ; RV64I-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV64I-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x11 + ; RV64I-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[COPY]](p0), [[COPY1]](p0) + ; RV64I-NEXT: PseudoRET + call void @llvm.va_copy(ptr %dest_list, ptr %src_list) + ret void +} diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vararg.ll b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vararg.ll index e03ef07b5fc0..ff30ebd3a8c7 100644 --- a/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vararg.ll +++ b/llvm/test/CodeGen/RISCV/GlobalISel/irtranslator/vararg.ll @@ -1,26 +1,27 @@ ; NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 3 ; RUN: llc -mtriple=riscv32 -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=ILP32 %s +; RUN: | FileCheck -check-prefixes=RV32,ILP32 %s ; RUN: llc -mtriple=riscv32 -global-isel -stop-after=irtranslator -mattr=+d -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=RV32D-ILP32 %s +; RUN: | FileCheck -check-prefixes=RV32,RV32D-ILP32 %s ; RUN: llc -mtriple=riscv32 -global-isel -stop-after=irtranslator -mattr=+d -target-abi ilp32f \ ; RUN: -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=RV32D-ILP32F %s +; RUN: | FileCheck -check-prefixes=RV32,RV32D-ILP32F %s ; RUN: llc -mtriple=riscv32 -global-isel -stop-after=irtranslator -mattr=+d -target-abi ilp32d \ ; RUN: -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=RV32D-ILP32D %s +; RUN: | FileCheck -check-prefixes=RV32,RV32D-ILP32D %s ; RUN: llc -mtriple=riscv64 -global-isel -stop-after=irtranslator -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=LP64 %s +; RUN: | FileCheck -check-prefixes=RV64,LP64 %s ; RUN: llc -mtriple=riscv64 -global-isel -stop-after=irtranslator -mattr=+d -target-abi lp64f \ ; RUN: -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=LP64F %s +; RUN: | FileCheck -check-prefixes=RV64,LP64F %s ; RUN: llc -mtriple=riscv64 -global-isel -stop-after=irtranslator -mattr=+d -target-abi lp64d \ ; RUN: -verify-machineinstrs < %s \ -; RUN: | FileCheck -check-prefixes=LP64D %s +; RUN: | FileCheck -check-prefixes=RV64,LP64D %s ; The same vararg calling convention is used for ilp32/ilp32f/ilp32d and for -; lp64/lp64f/lp64d. Different CHECK lines are required for RV32D due to slight -; codegen differences due to the way the f64 load operations are lowered. +; lp64/lp64f/lp64d. Different CHECK lines are required due to slight +; codegen differences due to the way the f64 load operations are lowered and +; because the PseudoCALL specifies the calling convention. ; The nounwind attribute is omitted for some of the tests, to check that CFI ; directives are correctly generated. @@ -29,7 +30,505 @@ declare void @llvm.va_end(ptr) declare void @notdead(ptr) -declare i32 @va1(ptr %fmt, ...) +; Although frontends are recommended to not generate va_arg due to the lack of +; support for aggregate types, we test simple cases here to ensure they are +; lowered correctly + +define i32 @va1(ptr %fmt, ...) { + ; RV32-LABEL: name: va1 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (p0) from %ir.va) + ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[LOAD]], [[C1]](s32) + ; RV32-NEXT: G_STORE [[PTR_ADD7]](p0), [[FRAME_INDEX1]](p0) :: (store (p0) into %ir.va) + ; RV32-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[LOAD]](p0) :: (load (s32) from %ir.argp.cur) + ; RV32-NEXT: $x10 = COPY [[LOAD1]](s32) + ; RV32-NEXT: PseudoRET implicit $x10 + ; + ; RV64-LABEL: name: va1 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (p0) from %ir.va, align 4) + ; RV64-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4 + ; RV64-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[LOAD]], [[C1]](s64) + ; RV64-NEXT: G_STORE [[PTR_ADD7]](p0), [[FRAME_INDEX1]](p0) :: (store (p0) into %ir.va, align 4) + ; RV64-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[LOAD]](p0) :: (load (s32) from %ir.argp.cur) + ; RV64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LOAD1]](s32) + ; RV64-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load ptr, ptr %va, align 4 + %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 + store ptr %argp.next, ptr %va, align 4 + %1 = load i32, ptr %argp.cur, align 4 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + +; Ensure the adjustment when restoring the stack pointer using the frame +; pointer is correct +define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind { + ; ILP32-LABEL: name: va1_va_arg_alloca + ; ILP32: bb.1 (%ir-block.0): + ; ILP32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; ILP32-NEXT: {{ $}} + ; ILP32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; ILP32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; ILP32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; ILP32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; ILP32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; ILP32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; ILP32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; ILP32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; ILP32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; ILP32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; ILP32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; ILP32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; ILP32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; ILP32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; ILP32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; ILP32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; ILP32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; ILP32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; ILP32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; ILP32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; ILP32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; ILP32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; ILP32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; ILP32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; ILP32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; ILP32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; ILP32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; ILP32-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; ILP32-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[VAARG]], [[C1]] + ; ILP32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 15 + ; ILP32-NEXT: [[ADD:%[0-9]+]]:_(s32) = nuw G_ADD [[MUL]], [[C2]] + ; ILP32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -16 + ; ILP32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C3]] + ; ILP32-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s32), 1 + ; ILP32-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; ILP32-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; ILP32-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32_lp64, implicit-def $x1, implicit $x10 + ; ILP32-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; ILP32-NEXT: $x10 = COPY [[VAARG]](s32) + ; ILP32-NEXT: PseudoRET implicit $x10 + ; + ; RV32D-ILP32-LABEL: name: va1_va_arg_alloca + ; RV32D-ILP32: bb.1 (%ir-block.0): + ; RV32D-ILP32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32D-ILP32-NEXT: {{ $}} + ; RV32D-ILP32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32D-ILP32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32D-ILP32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32D-ILP32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32D-ILP32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32D-ILP32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32D-ILP32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32D-ILP32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32D-ILP32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32D-ILP32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32D-ILP32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32D-ILP32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32D-ILP32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32D-ILP32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32D-ILP32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32D-ILP32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32D-ILP32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32D-ILP32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; RV32D-ILP32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32D-ILP32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32D-ILP32-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[VAARG]], [[C1]] + ; RV32D-ILP32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 15 + ; RV32D-ILP32-NEXT: [[ADD:%[0-9]+]]:_(s32) = nuw G_ADD [[MUL]], [[C2]] + ; RV32D-ILP32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -16 + ; RV32D-ILP32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C3]] + ; RV32D-ILP32-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s32), 1 + ; RV32D-ILP32-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; RV32D-ILP32-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10 + ; RV32D-ILP32-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32-NEXT: $x10 = COPY [[VAARG]](s32) + ; RV32D-ILP32-NEXT: PseudoRET implicit $x10 + ; + ; RV32D-ILP32F-LABEL: name: va1_va_arg_alloca + ; RV32D-ILP32F: bb.1 (%ir-block.0): + ; RV32D-ILP32F-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32D-ILP32F-NEXT: {{ $}} + ; RV32D-ILP32F-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32D-ILP32F-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32D-ILP32F-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32D-ILP32F-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32D-ILP32F-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32D-ILP32F-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32D-ILP32F-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32D-ILP32F-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32D-ILP32F-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32D-ILP32F-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32D-ILP32F-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; RV32D-ILP32F-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32D-ILP32F-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32D-ILP32F-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32F-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[VAARG]], [[C1]] + ; RV32D-ILP32F-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 15 + ; RV32D-ILP32F-NEXT: [[ADD:%[0-9]+]]:_(s32) = nuw G_ADD [[MUL]], [[C2]] + ; RV32D-ILP32F-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -16 + ; RV32D-ILP32F-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C3]] + ; RV32D-ILP32F-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s32), 1 + ; RV32D-ILP32F-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32F-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; RV32D-ILP32F-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10 + ; RV32D-ILP32F-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32F-NEXT: $x10 = COPY [[VAARG]](s32) + ; RV32D-ILP32F-NEXT: PseudoRET implicit $x10 + ; + ; RV32D-ILP32D-LABEL: name: va1_va_arg_alloca + ; RV32D-ILP32D: bb.1 (%ir-block.0): + ; RV32D-ILP32D-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32D-ILP32D-NEXT: {{ $}} + ; RV32D-ILP32D-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32D-ILP32D-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32D-ILP32D-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32D-ILP32D-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32D-ILP32D-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32D-ILP32D-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32D-ILP32D-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32D-ILP32D-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32D-ILP32D-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32D-ILP32D-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32D-ILP32D-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 1 + ; RV32D-ILP32D-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32D-ILP32D-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32D-ILP32D-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32D-NEXT: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[VAARG]], [[C1]] + ; RV32D-ILP32D-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 15 + ; RV32D-ILP32D-NEXT: [[ADD:%[0-9]+]]:_(s32) = nuw G_ADD [[MUL]], [[C2]] + ; RV32D-ILP32D-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 -16 + ; RV32D-ILP32D-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C3]] + ; RV32D-ILP32D-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s32), 1 + ; RV32D-ILP32D-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32D-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; RV32D-ILP32D-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10 + ; RV32D-ILP32D-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32D-NEXT: $x10 = COPY [[VAARG]](s32) + ; RV32D-ILP32D-NEXT: PseudoRET implicit $x10 + ; + ; LP64-LABEL: name: va1_va_arg_alloca + ; LP64: bb.1 (%ir-block.0): + ; LP64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; LP64-NEXT: {{ $}} + ; LP64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; LP64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; LP64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; LP64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; LP64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; LP64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; LP64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; LP64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; LP64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; LP64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; LP64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; LP64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; LP64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; LP64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; LP64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; LP64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; LP64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; LP64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; LP64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; LP64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; LP64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; LP64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; LP64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; LP64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; LP64-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; LP64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; LP64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; LP64-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[VAARG]](s32) + ; LP64-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ZEXT]], [[C1]] + ; LP64-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 15 + ; LP64-NEXT: [[ADD:%[0-9]+]]:_(s64) = nuw G_ADD [[MUL]], [[C2]] + ; LP64-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16 + ; LP64-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C3]] + ; LP64-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 1 + ; LP64-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; LP64-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; LP64-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32_lp64, implicit-def $x1, implicit $x10 + ; LP64-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; LP64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[VAARG]](s32) + ; LP64-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; LP64-NEXT: PseudoRET implicit $x10 + ; + ; LP64F-LABEL: name: va1_va_arg_alloca + ; LP64F: bb.1 (%ir-block.0): + ; LP64F-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; LP64F-NEXT: {{ $}} + ; LP64F-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; LP64F-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; LP64F-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; LP64F-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; LP64F-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; LP64F-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; LP64F-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; LP64F-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; LP64F-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; LP64F-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; LP64F-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; LP64F-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; LP64F-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; LP64F-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; LP64F-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; LP64F-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; LP64F-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; LP64F-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; LP64F-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; LP64F-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; LP64F-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; LP64F-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; LP64F-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; LP64F-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; LP64F-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; LP64F-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; LP64F-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; LP64F-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64F-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[VAARG]](s32) + ; LP64F-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ZEXT]], [[C1]] + ; LP64F-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 15 + ; LP64F-NEXT: [[ADD:%[0-9]+]]:_(s64) = nuw G_ADD [[MUL]], [[C2]] + ; LP64F-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16 + ; LP64F-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C3]] + ; LP64F-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 1 + ; LP64F-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; LP64F-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; LP64F-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10 + ; LP64F-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; LP64F-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[VAARG]](s32) + ; LP64F-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; LP64F-NEXT: PseudoRET implicit $x10 + ; + ; LP64D-LABEL: name: va1_va_arg_alloca + ; LP64D: bb.1 (%ir-block.0): + ; LP64D-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; LP64D-NEXT: {{ $}} + ; LP64D-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; LP64D-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; LP64D-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; LP64D-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; LP64D-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; LP64D-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; LP64D-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; LP64D-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; LP64D-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; LP64D-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; LP64D-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; LP64D-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; LP64D-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; LP64D-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; LP64D-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; LP64D-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; LP64D-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; LP64D-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; LP64D-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; LP64D-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; LP64D-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; LP64D-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; LP64D-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; LP64D-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; LP64D-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 1 + ; LP64D-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; LP64D-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; LP64D-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64D-NEXT: [[ZEXT:%[0-9]+]]:_(s64) = G_ZEXT [[VAARG]](s32) + ; LP64D-NEXT: [[MUL:%[0-9]+]]:_(s64) = G_MUL [[ZEXT]], [[C1]] + ; LP64D-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 15 + ; LP64D-NEXT: [[ADD:%[0-9]+]]:_(s64) = nuw G_ADD [[MUL]], [[C2]] + ; LP64D-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 -16 + ; LP64D-NEXT: [[AND:%[0-9]+]]:_(s64) = G_AND [[ADD]], [[C3]] + ; LP64D-NEXT: [[DYN_STACKALLOC:%[0-9]+]]:_(p0) = G_DYN_STACKALLOC [[AND]](s64), 1 + ; LP64D-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; LP64D-NEXT: $x10 = COPY [[DYN_STACKALLOC]](p0) + ; LP64D-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10 + ; LP64D-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; LP64D-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[VAARG]](s32) + ; LP64D-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; LP64D-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + %2 = alloca i8, i32 %1 + call void @notdead(ptr %2) + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + + +define i32 @va1_va_arg(ptr %fmt, ...) nounwind { + ; RV32-LABEL: name: va1_va_arg + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32-NEXT: $x10 = COPY [[VAARG]](s32) + ; RV32-NEXT: PseudoRET implicit $x10 + ; + ; RV64-LABEL: name: va1_va_arg + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[VAARG]](s32) + ; RV64-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} define void @va1_caller() nounwind { ; ILP32-LABEL: name: va1_caller @@ -150,7 +649,188 @@ define void @va1_caller() nounwind { ; Ensure that 2x xlen size+alignment varargs are accessed via an "aligned" ; register pair (where the first register is even-numbered). -declare i64 @va2(ptr %fmt, ...) nounwind +define i64 @va2(ptr %fmt, ...) nounwind { + ; RV32-LABEL: name: va2 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 7 + ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -8 + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (s32) from %ir.va) + ; RV32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD]], [[C1]] + ; RV32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]] + ; RV32-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[ADD]](s32) + ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8 + ; RV32-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[INTTOPTR]], [[C3]](s32) + ; RV32-NEXT: G_STORE [[PTR_ADD7]](p0), [[FRAME_INDEX1]](p0) :: (store (p0) into %ir.va) + ; RV32-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s32) + ; RV32-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[INTTOPTR1]](p0) :: (load (s64) from %ir.3) + ; RV32-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[LOAD1]](s64) + ; RV32-NEXT: $x10 = COPY [[UV]](s32) + ; RV32-NEXT: $x11 = COPY [[UV1]](s32) + ; RV32-NEXT: PseudoRET implicit $x10, implicit $x11 + ; + ; RV64-LABEL: name: va2 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; RV64-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 7 + ; RV64-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -8 + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (s32) from %ir.va) + ; RV64-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD]], [[C1]] + ; RV64-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]] + ; RV64-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[ADD]](s32) + ; RV64-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[INTTOPTR]], [[C3]](s64) + ; RV64-NEXT: G_STORE [[PTR_ADD7]](p0), [[FRAME_INDEX1]](p0) :: (store (p0) into %ir.va, align 4) + ; RV64-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s32) + ; RV64-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[INTTOPTR1]](p0) :: (load (s64) from %ir.3) + ; RV64-NEXT: $x10 = COPY [[LOAD1]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load i32, ptr %va, align 4 + %1 = add i32 %argp.cur, 7 + %2 = and i32 %1, -8 + %argp.cur.aligned = inttoptr i32 %1 to ptr + %argp.next = getelementptr inbounds i8, ptr %argp.cur.aligned, i32 8 + store ptr %argp.next, ptr %va, align 4 + %3 = inttoptr i32 %2 to ptr + %4 = load double, ptr %3, align 8 + %5 = bitcast double %4 to i64 + call void @llvm.va_end(ptr %va) + ret i64 %5 +} + +define i64 @va2_va_arg(ptr %fmt, ...) nounwind { + ; RV32-LABEL: name: va2_va_arg + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[VAARG:%[0-9]+]]:_(s64) = G_VAARG [[FRAME_INDEX1]](p0), 8 + ; RV32-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[VAARG]](s64) + ; RV32-NEXT: $x10 = COPY [[UV]](s32) + ; RV32-NEXT: $x11 = COPY [[UV1]](s32) + ; RV32-NEXT: PseudoRET implicit $x10, implicit $x11 + ; + ; RV64-LABEL: name: va2_va_arg + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[VAARG:%[0-9]+]]:_(s64) = G_VAARG [[FRAME_INDEX1]](p0), 8 + ; RV64-NEXT: $x10 = COPY [[VAARG]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, double + call void @llvm.va_end(ptr %va) + %2 = bitcast double %1 to i64 + ret i64 %2 +} define void @va2_caller() nounwind { ; ILP32-LABEL: name: va2_caller @@ -259,7 +939,186 @@ define void @va2_caller() nounwind { ; On RV32, Ensure a named 2*xlen argument is passed in a1 and a2, while the ; vararg double is passed in a4 and a5 (rather than a3 and a4) -declare i64 @va3(i32 %a, i64 %b, ...) nounwind +define i64 @va3(i32 %a, i64 %b, ...) nounwind { + ; RV32-LABEL: name: va3 + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32) + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 7 + ; RV32-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -8 + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (s32) from %ir.va) + ; RV32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD]], [[C1]] + ; RV32-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]] + ; RV32-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[ADD]](s32) + ; RV32-NEXT: [[C3:%[0-9]+]]:_(s32) = G_CONSTANT i32 8 + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[INTTOPTR]], [[C3]](s32) + ; RV32-NEXT: G_STORE [[PTR_ADD5]](p0), [[FRAME_INDEX1]](p0) :: (store (p0) into %ir.va) + ; RV32-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s32) + ; RV32-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[INTTOPTR1]](p0) :: (load (s64) from %ir.3) + ; RV32-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[MV]], [[LOAD1]] + ; RV32-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ADD1]](s64) + ; RV32-NEXT: $x10 = COPY [[UV]](s32) + ; RV32-NEXT: $x11 = COPY [[UV1]](s32) + ; RV32-NEXT: PseudoRET implicit $x10, implicit $x11 + ; + ; RV64-LABEL: name: va3 + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; RV64-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64) + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.0, align 16) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.0 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.0 + 16, align 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.0 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.0 + 32, align 16) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.0 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 7 + ; RV64-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 -8 + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[LOAD:%[0-9]+]]:_(s32) = G_LOAD [[FRAME_INDEX1]](p0) :: (dereferenceable load (s32) from %ir.va) + ; RV64-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[LOAD]], [[C1]] + ; RV64-NEXT: [[AND:%[0-9]+]]:_(s32) = G_AND [[ADD]], [[C2]] + ; RV64-NEXT: [[INTTOPTR:%[0-9]+]]:_(p0) = G_INTTOPTR [[ADD]](s32) + ; RV64-NEXT: [[C3:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[INTTOPTR]], [[C3]](s64) + ; RV64-NEXT: G_STORE [[PTR_ADD6]](p0), [[FRAME_INDEX1]](p0) :: (store (p0) into %ir.va, align 4) + ; RV64-NEXT: [[INTTOPTR1:%[0-9]+]]:_(p0) = G_INTTOPTR [[AND]](s32) + ; RV64-NEXT: [[LOAD1:%[0-9]+]]:_(s64) = G_LOAD [[INTTOPTR1]](p0) :: (load (s64) from %ir.3) + ; RV64-NEXT: [[ADD1:%[0-9]+]]:_(s64) = G_ADD [[COPY1]], [[LOAD1]] + ; RV64-NEXT: $x10 = COPY [[ADD1]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load i32, ptr %va, align 4 + %1 = add i32 %argp.cur, 7 + %2 = and i32 %1, -8 + %argp.cur.aligned = inttoptr i32 %1 to ptr + %argp.next = getelementptr inbounds i8, ptr %argp.cur.aligned, i32 8 + store ptr %argp.next, ptr %va, align 4 + %3 = inttoptr i32 %2 to ptr + %4 = load double, ptr %3, align 8 + call void @llvm.va_end(ptr %va) + %5 = bitcast double %4 to i64 + %6 = add i64 %b, %5 + ret i64 %6 +} + +define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind { + ; RV32-LABEL: name: va3_va_arg + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: [[MV:%[0-9]+]]:_(s64) = G_MERGE_VALUES [[COPY1]](s32), [[COPY2]](s32) + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[VAARG:%[0-9]+]]:_(s64) = G_VAARG [[FRAME_INDEX1]](p0), 8 + ; RV32-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[MV]], [[VAARG]] + ; RV32-NEXT: [[UV:%[0-9]+]]:_(s32), [[UV1:%[0-9]+]]:_(s32) = G_UNMERGE_VALUES [[ADD]](s64) + ; RV32-NEXT: $x10 = COPY [[UV]](s32) + ; RV32-NEXT: $x11 = COPY [[UV1]](s32) + ; RV32-NEXT: PseudoRET implicit $x10, implicit $x11 + ; + ; RV64-LABEL: name: va3_va_arg + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; RV64-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64) + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.0, align 16) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.0 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.0 + 16, align 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.0 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.0 + 32, align 16) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.0 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[VAARG:%[0-9]+]]:_(s64) = G_VAARG [[FRAME_INDEX1]](p0), 8 + ; RV64-NEXT: [[ADD:%[0-9]+]]:_(s64) = G_ADD [[COPY1]], [[VAARG]] + ; RV64-NEXT: $x10 = COPY [[ADD]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, double + call void @llvm.va_end(ptr %va) + %2 = bitcast double %1 to i64 + %3 = add i64 %b, %2 + ret i64 %3 +} define void @va3_caller() nounwind { ; ILP32-LABEL: name: va3_caller @@ -391,3 +1250,532 @@ define void @va3_caller() nounwind { } declare void @llvm.va_copy(ptr, ptr) + +define i32 @va4_va_copy(i32 %argno, ...) nounwind { + ; ILP32-LABEL: name: va4_va_copy + ; ILP32: bb.1 (%ir-block.0): + ; ILP32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; ILP32-NEXT: {{ $}} + ; ILP32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; ILP32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; ILP32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; ILP32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; ILP32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; ILP32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; ILP32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; ILP32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; ILP32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; ILP32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; ILP32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; ILP32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; ILP32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; ILP32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; ILP32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; ILP32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; ILP32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; ILP32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; ILP32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; ILP32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; ILP32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; ILP32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; ILP32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; ILP32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; ILP32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; ILP32-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; ILP32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.vargs) + ; ILP32-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; ILP32-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; ILP32-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs) + ; ILP32-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; ILP32-NEXT: $x10 = COPY [[LOAD]](p0) + ; ILP32-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32_lp64, implicit-def $x1, implicit $x10 + ; ILP32-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; ILP32-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; ILP32-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; ILP32-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; ILP32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; ILP32-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; ILP32-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; ILP32-NEXT: $x10 = COPY [[ADD2]](s32) + ; ILP32-NEXT: PseudoRET implicit $x10 + ; + ; RV32D-ILP32-LABEL: name: va4_va_copy + ; RV32D-ILP32: bb.1 (%ir-block.0): + ; RV32D-ILP32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32D-ILP32-NEXT: {{ $}} + ; RV32D-ILP32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32D-ILP32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32D-ILP32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32D-ILP32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32D-ILP32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32D-ILP32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32D-ILP32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32D-ILP32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32D-ILP32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32D-ILP32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32D-ILP32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32D-ILP32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32D-ILP32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32D-ILP32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32D-ILP32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32D-ILP32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32D-ILP32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32D-ILP32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32D-ILP32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; RV32D-ILP32-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; RV32D-ILP32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.vargs) + ; RV32D-ILP32-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; RV32D-ILP32-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs) + ; RV32D-ILP32-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32-NEXT: $x10 = COPY [[LOAD]](p0) + ; RV32D-ILP32-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10 + ; RV32D-ILP32-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; RV32D-ILP32-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; RV32D-ILP32-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; RV32D-ILP32-NEXT: $x10 = COPY [[ADD2]](s32) + ; RV32D-ILP32-NEXT: PseudoRET implicit $x10 + ; + ; RV32D-ILP32F-LABEL: name: va4_va_copy + ; RV32D-ILP32F: bb.1 (%ir-block.0): + ; RV32D-ILP32F-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32D-ILP32F-NEXT: {{ $}} + ; RV32D-ILP32F-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32D-ILP32F-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32D-ILP32F-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32D-ILP32F-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32D-ILP32F-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32D-ILP32F-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32D-ILP32F-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32D-ILP32F-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32D-ILP32F-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32D-ILP32F-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32D-ILP32F-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32D-ILP32F-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32D-ILP32F-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; RV32D-ILP32F-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; RV32D-ILP32F-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.vargs) + ; RV32D-ILP32F-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32F-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; RV32D-ILP32F-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs) + ; RV32D-ILP32F-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32F-NEXT: $x10 = COPY [[LOAD]](p0) + ; RV32D-ILP32F-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10 + ; RV32D-ILP32F-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32F-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32F-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32F-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32F-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; RV32D-ILP32F-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; RV32D-ILP32F-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; RV32D-ILP32F-NEXT: $x10 = COPY [[ADD2]](s32) + ; RV32D-ILP32F-NEXT: PseudoRET implicit $x10 + ; + ; RV32D-ILP32D-LABEL: name: va4_va_copy + ; RV32D-ILP32D: bb.1 (%ir-block.0): + ; RV32D-ILP32D-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32D-ILP32D-NEXT: {{ $}} + ; RV32D-ILP32D-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32D-ILP32D-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32D-ILP32D-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32D-ILP32D-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32D-ILP32D-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32D-ILP32D-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32D-ILP32D-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32D-ILP32D-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32D-ILP32D-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32D-ILP32D-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32D-ILP32D-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32D-ILP32D-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32D-ILP32D-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; RV32D-ILP32D-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; RV32D-ILP32D-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.vargs) + ; RV32D-ILP32D-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32D-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; RV32D-ILP32D-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs) + ; RV32D-ILP32D-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32D-NEXT: $x10 = COPY [[LOAD]](p0) + ; RV32D-ILP32D-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10 + ; RV32D-ILP32D-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; RV32D-ILP32D-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32D-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32D-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32D-ILP32D-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; RV32D-ILP32D-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; RV32D-ILP32D-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; RV32D-ILP32D-NEXT: $x10 = COPY [[ADD2]](s32) + ; RV32D-ILP32D-NEXT: PseudoRET implicit $x10 + ; + ; LP64-LABEL: name: va4_va_copy + ; LP64: bb.1 (%ir-block.0): + ; LP64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; LP64-NEXT: {{ $}} + ; LP64-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; LP64-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64) + ; LP64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; LP64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; LP64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; LP64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; LP64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; LP64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; LP64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; LP64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; LP64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; LP64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; LP64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; LP64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; LP64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; LP64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; LP64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; LP64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; LP64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; LP64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; LP64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; LP64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; LP64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; LP64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; LP64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; LP64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; LP64-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; LP64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.vargs) + ; LP64-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; LP64-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs, align 4) + ; LP64-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; LP64-NEXT: $x10 = COPY [[LOAD]](p0) + ; LP64-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32_lp64, implicit-def $x1, implicit $x10 + ; LP64-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; LP64-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; LP64-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; LP64-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; LP64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD2]](s32) + ; LP64-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; LP64-NEXT: PseudoRET implicit $x10 + ; + ; LP64F-LABEL: name: va4_va_copy + ; LP64F: bb.1 (%ir-block.0): + ; LP64F-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; LP64F-NEXT: {{ $}} + ; LP64F-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; LP64F-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64) + ; LP64F-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; LP64F-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; LP64F-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; LP64F-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; LP64F-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; LP64F-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; LP64F-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; LP64F-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; LP64F-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; LP64F-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; LP64F-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; LP64F-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; LP64F-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; LP64F-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; LP64F-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; LP64F-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; LP64F-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; LP64F-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; LP64F-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; LP64F-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; LP64F-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; LP64F-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; LP64F-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; LP64F-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; LP64F-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; LP64F-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.vargs) + ; LP64F-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64F-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; LP64F-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs, align 4) + ; LP64F-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; LP64F-NEXT: $x10 = COPY [[LOAD]](p0) + ; LP64F-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32f_lp64f, implicit-def $x1, implicit $x10 + ; LP64F-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; LP64F-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64F-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64F-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64F-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; LP64F-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; LP64F-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; LP64F-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD2]](s32) + ; LP64F-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; LP64F-NEXT: PseudoRET implicit $x10 + ; + ; LP64D-LABEL: name: va4_va_copy + ; LP64D: bb.1 (%ir-block.0): + ; LP64D-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; LP64D-NEXT: {{ $}} + ; LP64D-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; LP64D-NEXT: [[TRUNC:%[0-9]+]]:_(s32) = G_TRUNC [[COPY]](s64) + ; LP64D-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; LP64D-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; LP64D-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; LP64D-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; LP64D-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; LP64D-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; LP64D-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; LP64D-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; LP64D-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; LP64D-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; LP64D-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; LP64D-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; LP64D-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; LP64D-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; LP64D-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; LP64D-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; LP64D-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; LP64D-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; LP64D-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; LP64D-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; LP64D-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; LP64D-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; LP64D-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; LP64D-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.vargs + ; LP64D-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.wargs + ; LP64D-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.vargs) + ; LP64D-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64D-NEXT: G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), [[FRAME_INDEX2]](p0), [[FRAME_INDEX1]](p0) + ; LP64D-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.wargs, align 4) + ; LP64D-NEXT: ADJCALLSTACKDOWN 0, 0, implicit-def $x2, implicit $x2 + ; LP64D-NEXT: $x10 = COPY [[LOAD]](p0) + ; LP64D-NEXT: PseudoCALL target-flags(riscv-plt) @notdead, csr_ilp32d_lp64d, implicit-def $x1, implicit $x10 + ; LP64D-NEXT: ADJCALLSTACKUP 0, 0, implicit-def $x2, implicit $x2 + ; LP64D-NEXT: [[VAARG1:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64D-NEXT: [[VAARG2:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64D-NEXT: [[VAARG3:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; LP64D-NEXT: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[VAARG1]], [[VAARG]] + ; LP64D-NEXT: [[ADD1:%[0-9]+]]:_(s32) = G_ADD [[ADD]], [[VAARG2]] + ; LP64D-NEXT: [[ADD2:%[0-9]+]]:_(s32) = G_ADD [[ADD1]], [[VAARG3]] + ; LP64D-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[ADD2]](s32) + ; LP64D-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; LP64D-NEXT: PseudoRET implicit $x10 + %vargs = alloca ptr + %wargs = alloca ptr + call void @llvm.va_start(ptr %vargs) + %1 = va_arg ptr %vargs, i32 + call void @llvm.va_copy(ptr %wargs, ptr %vargs) + %2 = load ptr, ptr %wargs, align 4 + call void @notdead(ptr %2) + %3 = va_arg ptr %vargs, i32 + %4 = va_arg ptr %vargs, i32 + %5 = va_arg ptr %vargs, i32 + call void @llvm.va_end(ptr %vargs) + call void @llvm.va_end(ptr %wargs) + %add1 = add i32 %3, %1 + %add2 = add i32 %add1, %4 + %add3 = add i32 %add2, %5 + ret i32 %add3 +} + +; A function with no fixed arguments is not valid C, but can be +; specified in LLVM IR. We must ensure the vararg save area is +; still set up correctly. + +define i32 @va6_no_fixed_args(...) nounwind { + ; RV32-LABEL: name: va6_no_fixed_args + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(s32) = COPY $x10 + ; RV32-NEXT: G_STORE [[COPY]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.0, align 16) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: G_STORE [[COPY1]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.0 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.0 + 8, align 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.0 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.0 + 16, align 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.0 + 20) + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.0 + 24, align 8) + ; RV32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD6]](p0) :: (store (s32) into %fixed-stack.0 + 28) + ; RV32-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD6]], [[C]](s32) + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV32-NEXT: $x10 = COPY [[VAARG]](s32) + ; RV32-NEXT: PseudoRET implicit $x10 + ; + ; RV64-LABEL: name: va6_no_fixed_args + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(s64) = COPY $x10 + ; RV64-NEXT: G_STORE [[COPY]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.0, align 16) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: G_STORE [[COPY1]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.0 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.0 + 16, align 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.0 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.0 + 32, align 16) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.0 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.0 + 48, align 16) + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD6]](p0) :: (store (s64) into %fixed-stack.0 + 56) + ; RV64-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD6]], [[C]](s64) + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX1]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[VAARG:%[0-9]+]]:_(s32) = G_VAARG [[FRAME_INDEX1]](p0), 4 + ; RV64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[VAARG]](s32) + ; RV64-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + +; TODO: improve constant materialization of stack addresses + +define i32 @va_large_stack(ptr %fmt, ...) { + ; RV32-LABEL: name: va_large_stack + ; RV32: bb.1 (%ir-block.0): + ; RV32-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV32-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[COPY1:%[0-9]+]]:_(s32) = COPY $x11 + ; RV32-NEXT: G_STORE [[COPY1]](s32), [[FRAME_INDEX]](p0) :: (store (s32) into %fixed-stack.1) + ; RV32-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s32) + ; RV32-NEXT: [[COPY2:%[0-9]+]]:_(s32) = COPY $x12 + ; RV32-NEXT: G_STORE [[COPY2]](s32), [[PTR_ADD]](p0) :: (store (s32) into %fixed-stack.1 + 4) + ; RV32-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s32) + ; RV32-NEXT: [[COPY3:%[0-9]+]]:_(s32) = COPY $x13 + ; RV32-NEXT: G_STORE [[COPY3]](s32), [[PTR_ADD1]](p0) :: (store (s32) into %fixed-stack.1 + 8) + ; RV32-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s32) + ; RV32-NEXT: [[COPY4:%[0-9]+]]:_(s32) = COPY $x14 + ; RV32-NEXT: G_STORE [[COPY4]](s32), [[PTR_ADD2]](p0) :: (store (s32) into %fixed-stack.1 + 12) + ; RV32-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s32) + ; RV32-NEXT: [[COPY5:%[0-9]+]]:_(s32) = COPY $x15 + ; RV32-NEXT: G_STORE [[COPY5]](s32), [[PTR_ADD3]](p0) :: (store (s32) into %fixed-stack.1 + 16) + ; RV32-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s32) + ; RV32-NEXT: [[COPY6:%[0-9]+]]:_(s32) = COPY $x16 + ; RV32-NEXT: G_STORE [[COPY6]](s32), [[PTR_ADD4]](p0) :: (store (s32) into %fixed-stack.1 + 20) + ; RV32-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s32) + ; RV32-NEXT: [[COPY7:%[0-9]+]]:_(s32) = COPY $x17 + ; RV32-NEXT: G_STORE [[COPY7]](s32), [[PTR_ADD5]](p0) :: (store (s32) into %fixed-stack.1 + 24) + ; RV32-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s32) + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.large + ; RV32-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.va + ; RV32-NEXT: G_VASTART [[FRAME_INDEX2]](p0) :: (store (s32) into %ir.va) + ; RV32-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.va) + ; RV32-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; RV32-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[LOAD]], [[C1]](s32) + ; RV32-NEXT: G_STORE [[PTR_ADD7]](p0), [[FRAME_INDEX2]](p0) :: (store (p0) into %ir.va) + ; RV32-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[LOAD]](p0) :: (load (s32) from %ir.argp.cur) + ; RV32-NEXT: $x10 = COPY [[LOAD1]](s32) + ; RV32-NEXT: PseudoRET implicit $x10 + ; + ; RV64-LABEL: name: va_large_stack + ; RV64: bb.1 (%ir-block.0): + ; RV64-NEXT: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.1 + ; RV64-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; RV64-NEXT: [[COPY1:%[0-9]+]]:_(s64) = COPY $x11 + ; RV64-NEXT: G_STORE [[COPY1]](s64), [[FRAME_INDEX]](p0) :: (store (s64) into %fixed-stack.1) + ; RV64-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[FRAME_INDEX]], [[C]](s64) + ; RV64-NEXT: [[COPY2:%[0-9]+]]:_(s64) = COPY $x12 + ; RV64-NEXT: G_STORE [[COPY2]](s64), [[PTR_ADD]](p0) :: (store (s64) into %fixed-stack.1 + 8) + ; RV64-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD]], [[C]](s64) + ; RV64-NEXT: [[COPY3:%[0-9]+]]:_(s64) = COPY $x13 + ; RV64-NEXT: G_STORE [[COPY3]](s64), [[PTR_ADD1]](p0) :: (store (s64) into %fixed-stack.1 + 16) + ; RV64-NEXT: [[PTR_ADD2:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD1]], [[C]](s64) + ; RV64-NEXT: [[COPY4:%[0-9]+]]:_(s64) = COPY $x14 + ; RV64-NEXT: G_STORE [[COPY4]](s64), [[PTR_ADD2]](p0) :: (store (s64) into %fixed-stack.1 + 24) + ; RV64-NEXT: [[PTR_ADD3:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD2]], [[C]](s64) + ; RV64-NEXT: [[COPY5:%[0-9]+]]:_(s64) = COPY $x15 + ; RV64-NEXT: G_STORE [[COPY5]](s64), [[PTR_ADD3]](p0) :: (store (s64) into %fixed-stack.1 + 32) + ; RV64-NEXT: [[PTR_ADD4:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD3]], [[C]](s64) + ; RV64-NEXT: [[COPY6:%[0-9]+]]:_(s64) = COPY $x16 + ; RV64-NEXT: G_STORE [[COPY6]](s64), [[PTR_ADD4]](p0) :: (store (s64) into %fixed-stack.1 + 40) + ; RV64-NEXT: [[PTR_ADD5:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD4]], [[C]](s64) + ; RV64-NEXT: [[COPY7:%[0-9]+]]:_(s64) = COPY $x17 + ; RV64-NEXT: G_STORE [[COPY7]](s64), [[PTR_ADD5]](p0) :: (store (s64) into %fixed-stack.1 + 48) + ; RV64-NEXT: [[PTR_ADD6:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTR_ADD5]], [[C]](s64) + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0.large + ; RV64-NEXT: [[FRAME_INDEX2:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.1.va + ; RV64-NEXT: G_VASTART [[FRAME_INDEX2]](p0) :: (store (s64) into %ir.va) + ; RV64-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX2]](p0) :: (dereferenceable load (p0) from %ir.va, align 4) + ; RV64-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 4 + ; RV64-NEXT: [[PTR_ADD7:%[0-9]+]]:_(p0) = nuw G_PTR_ADD [[LOAD]], [[C1]](s64) + ; RV64-NEXT: G_STORE [[PTR_ADD7]](p0), [[FRAME_INDEX2]](p0) :: (store (p0) into %ir.va, align 4) + ; RV64-NEXT: [[LOAD1:%[0-9]+]]:_(s32) = G_LOAD [[LOAD]](p0) :: (load (s32) from %ir.argp.cur) + ; RV64-NEXT: [[ANYEXT:%[0-9]+]]:_(s64) = G_ANYEXT [[LOAD1]](s32) + ; RV64-NEXT: $x10 = COPY [[ANYEXT]](s64) + ; RV64-NEXT: PseudoRET implicit $x10 + %large = alloca [ 100000000 x i8 ] + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load ptr, ptr %va, align 4 + %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 + store ptr %argp.next, ptr %va, align 4 + %1 = load i32, ptr %argp.cur, align 4 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv32.mir new file mode 100644 index 000000000000..b67691d0ef06 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv32.mir @@ -0,0 +1,70 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - | FileCheck %s + +# On RISC-V, the MinStackArgumentAlignment is 1 and the ABI Alignment for p0 is +# greater than 1, so we will always generate code to adjust for this alignment. + +--- +name: va_arg_i32 +legalized: false +tracksRegLiveness: true +fixedStack: + - { id: 0, type: default, offset: 0, size: 4, alignment: 16, + isImmutable: true, isAliased: false } +stack: + - { id: 0, type: default, offset: 0, size: 4, alignment: 4 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; CHECK-LABEL: name: va_arg_i32 + ; CHECK: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load (p0)) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3 + ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s32) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -4 + ; CHECK-NEXT: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[PTR_ADD]], [[C1]](s32) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTRMASK]], [[C2]](s32) + ; CHECK-NEXT: G_STORE [[PTR_ADD1]](p0), [[FRAME_INDEX]](p0) :: (store (p0)) + ; CHECK-NEXT: PseudoRET + %0:_(p0) = G_FRAME_INDEX %stack.0 + %1:_(s32) = G_VAARG %0(p0), 4 + PseudoRET +... +--- +name: va_arg_ptr +legalized: false +tracksRegLiveness: true +fixedStack: + - { id: 0, type: default, offset: 0, size: 4, alignment: 16, + isImmutable: true, isAliased: false } +stack: + - { id: 0, type: default, offset: 0, size: 4, alignment: 4 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; CHECK-LABEL: name: va_arg_ptr + ; CHECK: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load (p0)) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s32) = G_CONSTANT i32 3 + ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s32) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s32) = G_CONSTANT i32 -4 + ; CHECK-NEXT: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[PTR_ADD]], [[C1]](s32) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s32) = G_CONSTANT i32 4 + ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTRMASK]], [[C2]](s32) + ; CHECK-NEXT: G_STORE [[PTR_ADD1]](p0), [[FRAME_INDEX]](p0) :: (store (p0)) + ; CHECK-NEXT: PseudoRET + %0:_(p0) = G_FRAME_INDEX %stack.0 + %1:_(p0) = G_VAARG %0(p0), 4 + PseudoRET +... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv64.mir new file mode 100644 index 000000000000..537c0fb14d26 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vaarg-rv64.mir @@ -0,0 +1,93 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - | FileCheck %s + +# On RISC-V, the MinStackArgumentAlignment is 1 and the ABI Alignment for p0 is +# greater than 1, so we will always generate code to adjust for this alignment. + +--- +name: va_arg_i32 +legalized: false +tracksRegLiveness: true +fixedStack: + - { id: 0, type: default, offset: 0, size: 8, alignment: 16, + isImmutable: true, isAliased: false } +stack: + - { id: 0, type: default, offset: 0, size: 8, alignment: 8 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + ; CHECK-LABEL: name: va_arg_i32 + ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load (p0)) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3 + ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s64) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -4 + ; CHECK-NEXT: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[PTR_ADD]], [[C1]](s64) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 4 + ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTRMASK]], [[C2]](s64) + ; CHECK-NEXT: G_STORE [[PTR_ADD1]](p0), [[FRAME_INDEX]](p0) :: (store (p0)) + ; CHECK-NEXT: PseudoRET + %0:_(p0) = G_FRAME_INDEX %stack.0 + %1:_(s32) = G_VAARG %0(p0), 4 + PseudoRET +... +--- +name: va_arg_i64 +legalized: false +tracksRegLiveness: true +fixedStack: + - { id: 0, type: default, offset: 0, size: 8, alignment: 16, + isImmutable: true, isAliased: false } +stack: + - { id: 0, type: default, offset: 0, size: 8, alignment: 8 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + ; CHECK-LABEL: name: va_arg_i64 + ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load (p0)) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3 + ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s64) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -4 + ; CHECK-NEXT: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[PTR_ADD]], [[C1]](s64) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTRMASK]], [[C2]](s64) + ; CHECK-NEXT: G_STORE [[PTR_ADD1]](p0), [[FRAME_INDEX]](p0) :: (store (p0)) + ; CHECK-NEXT: PseudoRET + %0:_(p0) = G_FRAME_INDEX %stack.0 + %1:_(s64) = G_VAARG %0(p0), 4 + PseudoRET +... +--- +name: va_arg_ptr +legalized: false +tracksRegLiveness: true +fixedStack: + - { id: 0, type: default, offset: 0, size: 8, alignment: 16, + isImmutable: true, isAliased: false } +stack: + - { id: 0, type: default, offset: 0, size: 8, alignment: 8 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + ; CHECK-LABEL: name: va_arg_ptr + ; CHECK: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[FRAME_INDEX]](p0) :: (load (p0)) + ; CHECK-NEXT: [[C:%[0-9]+]]:_(s64) = G_CONSTANT i64 3 + ; CHECK-NEXT: [[PTR_ADD:%[0-9]+]]:_(p0) = G_PTR_ADD [[LOAD]], [[C]](s64) + ; CHECK-NEXT: [[C1:%[0-9]+]]:_(s64) = G_CONSTANT i64 -4 + ; CHECK-NEXT: [[PTRMASK:%[0-9]+]]:_(p0) = G_PTRMASK [[PTR_ADD]], [[C1]](s64) + ; CHECK-NEXT: [[C2:%[0-9]+]]:_(s64) = G_CONSTANT i64 8 + ; CHECK-NEXT: [[PTR_ADD1:%[0-9]+]]:_(p0) = G_PTR_ADD [[PTRMASK]], [[C2]](s64) + ; CHECK-NEXT: G_STORE [[PTR_ADD1]](p0), [[FRAME_INDEX]](p0) :: (store (p0)) + ; CHECK-NEXT: PseudoRET + %0:_(p0) = G_FRAME_INDEX %stack.0 + %1:_(s64) = G_VAARG %0(p0), 4 + PseudoRET +... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vacopy.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vacopy.mir new file mode 100644 index 000000000000..f9eda1252937 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vacopy.mir @@ -0,0 +1,23 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - | FileCheck %s +# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - | FileCheck %s + +--- +name: test_va_copy +body: | + bb.1: + liveins: $x10, $x11 + + ; CHECK-LABEL: name: test_va_copy + ; CHECK: liveins: $x10, $x11 + ; CHECK-NEXT: {{ $}} + ; CHECK-NEXT: [[COPY:%[0-9]+]]:_(p0) = COPY $x10 + ; CHECK-NEXT: [[COPY1:%[0-9]+]]:_(p0) = COPY $x11 + ; CHECK-NEXT: [[LOAD:%[0-9]+]]:_(p0) = G_LOAD [[COPY1]](p0) :: (load (p0)) + ; CHECK-NEXT: G_STORE [[COPY]](p0), [[LOAD]](p0) :: (store (p0)) + ; CHECK-NEXT: PseudoRET + %0:_(p0) = COPY $x10 + %1:_(p0) = COPY $x11 + G_INTRINSIC_W_SIDE_EFFECTS intrinsic(@llvm.va_copy), %0(p0), %1(p0) + PseudoRET +... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv32.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv32.mir new file mode 100644 index 000000000000..181aa9cb9b31 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv32.mir @@ -0,0 +1,29 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=riscv32 -run-pass=legalizer %s -o - | FileCheck --check-prefix=RV32 %s + +--- +name: test_va_start +tracksRegLiveness: true +fixedStack: + - { id: 0, size: 4, alignment: 16, isImmutable: true } +stack: + - { id: 0, size: 4, alignment: 4 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + + ; RV32-LABEL: name: test_va_start + ; RV32: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV32-NEXT: {{ $}} + ; RV32-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; RV32-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 + ; RV32-NEXT: G_STORE [[FRAME_INDEX1]](p0), [[FRAME_INDEX]](p0) :: (store (s32)) + ; RV32-NEXT: PseudoRET + %8:_(p0) = G_FRAME_INDEX %stack.0 + G_VASTART %8(p0) :: (store (s32)) + PseudoRET + +... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv64.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv64.mir new file mode 100644 index 000000000000..60babc87150a --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer/legalize-vastart-rv64.mir @@ -0,0 +1,29 @@ +# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py +# RUN: llc -mtriple=riscv64 -run-pass=legalizer %s -o - | FileCheck --check-prefix=RV64 %s + +--- +name: test_va_start +tracksRegLiveness: true +fixedStack: + - { id: 0, size: 8, alignment: 16, isImmutable: true } +stack: + - { id: 0, size: 8, alignment: 8 } +machineFunctionInfo: + varArgsFrameIndex: -1 + varArgsSaveSize: 0 +body: | + bb.1: + liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + + ; RV64-LABEL: name: test_va_start + ; RV64: liveins: $x10, $x11, $x12, $x13, $x14, $x15, $x16, $x17 + ; RV64-NEXT: {{ $}} + ; RV64-NEXT: [[FRAME_INDEX:%[0-9]+]]:_(p0) = G_FRAME_INDEX %stack.0 + ; RV64-NEXT: [[FRAME_INDEX1:%[0-9]+]]:_(p0) = G_FRAME_INDEX %fixed-stack.0 + ; RV64-NEXT: G_STORE [[FRAME_INDEX1]](p0), [[FRAME_INDEX]](p0) :: (store (s64)) + ; RV64-NEXT: PseudoRET + %8:_(p0) = G_FRAME_INDEX %stack.0 + G_VASTART %8(p0) :: (store (s64)) + PseudoRET + +... diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll b/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll new file mode 100644 index 000000000000..501a3c0ce743 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/GlobalISel/vararg.ll @@ -0,0 +1,1008 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV32,ILP32 %s +; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel -mattr=+d -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV32,RV32D-ILP32 %s +; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel -mattr=+d -target-abi ilp32f \ +; RUN: -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV32,RV32D-ILP32F %s +; RUN: sed 's/iXLen/i32/g' %s | llc -mtriple=riscv32 -global-isel -mattr=+d -target-abi ilp32d \ +; RUN: -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV32,RV32D-ILP32D %s +; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -global-isel -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV64,LP64 %s +; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -global-isel -mattr=+d -target-abi lp64f \ +; RUN: -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV64,LP64F %s +; RUN: sed 's/iXLen/i64/g' %s | llc -mtriple=riscv64 -global-isel -mattr=+d -target-abi lp64d \ +; RUN: -verify-machineinstrs \ +; RUN: | FileCheck -check-prefixes=RV64,LP64D %s + +; The same vararg calling convention is used for ilp32/ilp32f/ilp32d and for +; lp64/lp64f/lp64d. Different CHECK lines are required due to slight +; codegen differences due to the way the f64 load operations are lowered and +; because the PseudoCALL specifies the calling convention. +; The nounwind attribute is omitted for some of the tests, to check that CFI +; directives are correctly generated. + +declare void @llvm.va_start(ptr) +declare void @llvm.va_end(ptr) + +declare void @notdead(ptr) + +; Although frontends are recommended to not generate va_arg due to the lack of +; support for aggregate types, we test simple cases here to ensure they are +; lowered correctly + +define i32 @va1(ptr %fmt, ...) { +; RV32-LABEL: va1: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -48 +; RV32-NEXT: .cfi_def_cfa_offset 48 +; RV32-NEXT: sw a1, 20(sp) +; RV32-NEXT: sw a2, 24(sp) +; RV32-NEXT: sw a3, 28(sp) +; RV32-NEXT: sw a4, 32(sp) +; RV32-NEXT: addi a0, sp, 20 +; RV32-NEXT: sw a0, 12(sp) +; RV32-NEXT: lw a0, 12(sp) +; RV32-NEXT: sw a5, 36(sp) +; RV32-NEXT: sw a6, 40(sp) +; RV32-NEXT: sw a7, 44(sp) +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 12(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: addi sp, sp, 48 +; RV32-NEXT: ret +; +; RV64-LABEL: va1: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -80 +; RV64-NEXT: .cfi_def_cfa_offset 80 +; RV64-NEXT: sd a1, 24(sp) +; RV64-NEXT: sd a2, 32(sp) +; RV64-NEXT: sd a3, 40(sp) +; RV64-NEXT: sd a4, 48(sp) +; RV64-NEXT: sd a5, 56(sp) +; RV64-NEXT: addi a0, sp, 24 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: lw a0, 12(sp) +; RV64-NEXT: lwu a1, 8(sp) +; RV64-NEXT: sd a6, 64(sp) +; RV64-NEXT: sd a7, 72(sp) +; RV64-NEXT: slli a0, a0, 32 +; RV64-NEXT: or a0, a0, a1 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: srli a2, a1, 32 +; RV64-NEXT: sw a1, 8(sp) +; RV64-NEXT: sw a2, 12(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: addi sp, sp, 80 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load ptr, ptr %va, align 4 + %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 + store ptr %argp.next, ptr %va, align 4 + %1 = load i32, ptr %argp.cur, align 4 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + +define i32 @va1_va_arg(ptr %fmt, ...) nounwind { +; RV32-LABEL: va1_va_arg: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -48 +; RV32-NEXT: sw a1, 20(sp) +; RV32-NEXT: sw a2, 24(sp) +; RV32-NEXT: sw a3, 28(sp) +; RV32-NEXT: sw a4, 32(sp) +; RV32-NEXT: sw a5, 36(sp) +; RV32-NEXT: sw a6, 40(sp) +; RV32-NEXT: sw a7, 44(sp) +; RV32-NEXT: addi a0, sp, 20 +; RV32-NEXT: sw a0, 12(sp) +; RV32-NEXT: lw a0, 12(sp) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 12(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: addi sp, sp, 48 +; RV32-NEXT: ret +; +; RV64-LABEL: va1_va_arg: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -80 +; RV64-NEXT: sd a1, 24(sp) +; RV64-NEXT: sd a2, 32(sp) +; RV64-NEXT: sd a3, 40(sp) +; RV64-NEXT: sd a4, 48(sp) +; RV64-NEXT: sd a5, 56(sp) +; RV64-NEXT: sd a6, 64(sp) +; RV64-NEXT: sd a7, 72(sp) +; RV64-NEXT: addi a0, sp, 24 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: sd a1, 8(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: addi sp, sp, 80 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + +; Ensure the adjustment when restoring the stack pointer using the frame +; pointer is correct +define i32 @va1_va_arg_alloca(ptr %fmt, ...) nounwind { +; RV32-LABEL: va1_va_arg_alloca: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -48 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s1, 4(sp) # 4-byte Folded Spill +; RV32-NEXT: addi s0, sp, 16 +; RV32-NEXT: sw a1, 4(s0) +; RV32-NEXT: sw a2, 8(s0) +; RV32-NEXT: sw a3, 12(s0) +; RV32-NEXT: sw a4, 16(s0) +; RV32-NEXT: sw a5, 20(s0) +; RV32-NEXT: sw a6, 24(s0) +; RV32-NEXT: sw a7, 28(s0) +; RV32-NEXT: addi a0, s0, 4 +; RV32-NEXT: sw a0, -16(s0) +; RV32-NEXT: lw a0, -16(s0) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, -16(s0) +; RV32-NEXT: lw s1, 0(a0) +; RV32-NEXT: addi a0, s1, 15 +; RV32-NEXT: andi a0, a0, -16 +; RV32-NEXT: sub a0, sp, a0 +; RV32-NEXT: mv sp, a0 +; RV32-NEXT: call notdead@plt +; RV32-NEXT: mv a0, s1 +; RV32-NEXT: addi sp, s0, -16 +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s1, 4(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 48 +; RV32-NEXT: ret +; +; RV64-LABEL: va1_va_arg_alloca: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -96 +; RV64-NEXT: sd ra, 24(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s0, 16(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s1, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: addi s0, sp, 32 +; RV64-NEXT: sd a1, 8(s0) +; RV64-NEXT: sd a2, 16(s0) +; RV64-NEXT: sd a3, 24(s0) +; RV64-NEXT: sd a4, 32(s0) +; RV64-NEXT: sd a5, 40(s0) +; RV64-NEXT: sd a6, 48(s0) +; RV64-NEXT: sd a7, 56(s0) +; RV64-NEXT: addi a0, s0, 8 +; RV64-NEXT: sd a0, -32(s0) +; RV64-NEXT: ld a0, -32(s0) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: sd a1, -32(s0) +; RV64-NEXT: lw s1, 0(a0) +; RV64-NEXT: slli a0, s1, 32 +; RV64-NEXT: srli a0, a0, 32 +; RV64-NEXT: addi a0, a0, 15 +; RV64-NEXT: andi a0, a0, -16 +; RV64-NEXT: sub a0, sp, a0 +; RV64-NEXT: mv sp, a0 +; RV64-NEXT: call notdead@plt +; RV64-NEXT: mv a0, s1 +; RV64-NEXT: addi sp, s0, -32 +; RV64-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s1, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 96 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + %2 = alloca i8, i32 %1 + call void @notdead(ptr %2) + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + +define void @va1_caller() nounwind { +; RV32-LABEL: va1_caller: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -16 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: lui a3, 261888 +; RV32-NEXT: li a4, 2 +; RV32-NEXT: li a2, 0 +; RV32-NEXT: call va1@plt +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 16 +; RV32-NEXT: ret +; +; LP64-LABEL: va1_caller: +; LP64: # %bb.0: +; LP64-NEXT: addi sp, sp, -16 +; LP64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; LP64-NEXT: lui a0, %hi(.LCPI3_0) +; LP64-NEXT: ld a1, %lo(.LCPI3_0)(a0) +; LP64-NEXT: li a2, 2 +; LP64-NEXT: call va1@plt +; LP64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; LP64-NEXT: addi sp, sp, 16 +; LP64-NEXT: ret +; +; LP64F-LABEL: va1_caller: +; LP64F: # %bb.0: +; LP64F-NEXT: addi sp, sp, -16 +; LP64F-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; LP64F-NEXT: li a0, 1023 +; LP64F-NEXT: slli a0, a0, 52 +; LP64F-NEXT: fmv.d.x fa5, a0 +; LP64F-NEXT: li a2, 2 +; LP64F-NEXT: fmv.x.d a1, fa5 +; LP64F-NEXT: call va1@plt +; LP64F-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; LP64F-NEXT: addi sp, sp, 16 +; LP64F-NEXT: ret +; +; LP64D-LABEL: va1_caller: +; LP64D: # %bb.0: +; LP64D-NEXT: addi sp, sp, -16 +; LP64D-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; LP64D-NEXT: li a0, 1023 +; LP64D-NEXT: slli a0, a0, 52 +; LP64D-NEXT: fmv.d.x fa5, a0 +; LP64D-NEXT: li a2, 2 +; LP64D-NEXT: fmv.x.d a1, fa5 +; LP64D-NEXT: call va1@plt +; LP64D-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; LP64D-NEXT: addi sp, sp, 16 +; LP64D-NEXT: ret + %1 = call i32 (ptr, ...) @va1(ptr undef, double 1.0, i32 2) + ret void +} + +; Ensure that 2x xlen size+alignment varargs are accessed via an "aligned" +; register pair (where the first register is even-numbered). + +define i64 @va2(ptr %fmt, ...) nounwind { +; ILP32-LABEL: va2: +; ILP32: # %bb.0: +; ILP32-NEXT: addi sp, sp, -48 +; ILP32-NEXT: sw a1, 20(sp) +; ILP32-NEXT: sw a2, 24(sp) +; ILP32-NEXT: sw a3, 28(sp) +; ILP32-NEXT: sw a4, 32(sp) +; ILP32-NEXT: addi a0, sp, 20 +; ILP32-NEXT: sw a0, 12(sp) +; ILP32-NEXT: lw a0, 12(sp) +; ILP32-NEXT: sw a5, 36(sp) +; ILP32-NEXT: sw a6, 40(sp) +; ILP32-NEXT: sw a7, 44(sp) +; ILP32-NEXT: addi a1, a0, 7 +; ILP32-NEXT: andi a1, a1, -8 +; ILP32-NEXT: addi a0, a0, 15 +; ILP32-NEXT: sw a0, 12(sp) +; ILP32-NEXT: lw a0, 0(a1) +; ILP32-NEXT: lw a1, 4(a1) +; ILP32-NEXT: addi sp, sp, 48 +; ILP32-NEXT: ret +; +; RV32D-ILP32-LABEL: va2: +; RV32D-ILP32: # %bb.0: +; RV32D-ILP32-NEXT: addi sp, sp, -48 +; RV32D-ILP32-NEXT: sw a1, 20(sp) +; RV32D-ILP32-NEXT: sw a2, 24(sp) +; RV32D-ILP32-NEXT: sw a3, 28(sp) +; RV32D-ILP32-NEXT: sw a4, 32(sp) +; RV32D-ILP32-NEXT: addi a0, sp, 20 +; RV32D-ILP32-NEXT: sw a0, 12(sp) +; RV32D-ILP32-NEXT: lw a0, 12(sp) +; RV32D-ILP32-NEXT: sw a5, 36(sp) +; RV32D-ILP32-NEXT: sw a6, 40(sp) +; RV32D-ILP32-NEXT: sw a7, 44(sp) +; RV32D-ILP32-NEXT: addi a1, a0, 7 +; RV32D-ILP32-NEXT: andi a1, a1, -8 +; RV32D-ILP32-NEXT: fld fa5, 0(a1) +; RV32D-ILP32-NEXT: addi a0, a0, 15 +; RV32D-ILP32-NEXT: sw a0, 12(sp) +; RV32D-ILP32-NEXT: fsd fa5, 0(sp) +; RV32D-ILP32-NEXT: lw a0, 0(sp) +; RV32D-ILP32-NEXT: lw a1, 4(sp) +; RV32D-ILP32-NEXT: addi sp, sp, 48 +; RV32D-ILP32-NEXT: ret +; +; RV32D-ILP32F-LABEL: va2: +; RV32D-ILP32F: # %bb.0: +; RV32D-ILP32F-NEXT: addi sp, sp, -48 +; RV32D-ILP32F-NEXT: sw a1, 20(sp) +; RV32D-ILP32F-NEXT: sw a2, 24(sp) +; RV32D-ILP32F-NEXT: sw a3, 28(sp) +; RV32D-ILP32F-NEXT: sw a4, 32(sp) +; RV32D-ILP32F-NEXT: addi a0, sp, 20 +; RV32D-ILP32F-NEXT: sw a0, 12(sp) +; RV32D-ILP32F-NEXT: lw a0, 12(sp) +; RV32D-ILP32F-NEXT: sw a5, 36(sp) +; RV32D-ILP32F-NEXT: sw a6, 40(sp) +; RV32D-ILP32F-NEXT: sw a7, 44(sp) +; RV32D-ILP32F-NEXT: addi a1, a0, 7 +; RV32D-ILP32F-NEXT: andi a1, a1, -8 +; RV32D-ILP32F-NEXT: fld fa5, 0(a1) +; RV32D-ILP32F-NEXT: addi a0, a0, 15 +; RV32D-ILP32F-NEXT: sw a0, 12(sp) +; RV32D-ILP32F-NEXT: fsd fa5, 0(sp) +; RV32D-ILP32F-NEXT: lw a0, 0(sp) +; RV32D-ILP32F-NEXT: lw a1, 4(sp) +; RV32D-ILP32F-NEXT: addi sp, sp, 48 +; RV32D-ILP32F-NEXT: ret +; +; RV32D-ILP32D-LABEL: va2: +; RV32D-ILP32D: # %bb.0: +; RV32D-ILP32D-NEXT: addi sp, sp, -48 +; RV32D-ILP32D-NEXT: sw a1, 20(sp) +; RV32D-ILP32D-NEXT: sw a2, 24(sp) +; RV32D-ILP32D-NEXT: sw a3, 28(sp) +; RV32D-ILP32D-NEXT: sw a4, 32(sp) +; RV32D-ILP32D-NEXT: addi a0, sp, 20 +; RV32D-ILP32D-NEXT: sw a0, 12(sp) +; RV32D-ILP32D-NEXT: lw a0, 12(sp) +; RV32D-ILP32D-NEXT: sw a5, 36(sp) +; RV32D-ILP32D-NEXT: sw a6, 40(sp) +; RV32D-ILP32D-NEXT: sw a7, 44(sp) +; RV32D-ILP32D-NEXT: addi a1, a0, 7 +; RV32D-ILP32D-NEXT: andi a1, a1, -8 +; RV32D-ILP32D-NEXT: fld fa5, 0(a1) +; RV32D-ILP32D-NEXT: addi a0, a0, 15 +; RV32D-ILP32D-NEXT: sw a0, 12(sp) +; RV32D-ILP32D-NEXT: fsd fa5, 0(sp) +; RV32D-ILP32D-NEXT: lw a0, 0(sp) +; RV32D-ILP32D-NEXT: lw a1, 4(sp) +; RV32D-ILP32D-NEXT: addi sp, sp, 48 +; RV32D-ILP32D-NEXT: ret +; +; RV64-LABEL: va2: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -80 +; RV64-NEXT: sd a1, 24(sp) +; RV64-NEXT: sd a2, 32(sp) +; RV64-NEXT: sd a3, 40(sp) +; RV64-NEXT: sd a4, 48(sp) +; RV64-NEXT: addi a0, sp, 24 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: sd a5, 56(sp) +; RV64-NEXT: sd a6, 64(sp) +; RV64-NEXT: sd a7, 72(sp) +; RV64-NEXT: addi a1, a0, 7 +; RV64-NEXT: andi a1, a1, -8 +; RV64-NEXT: addi a0, a0, 15 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 0(a1) +; RV64-NEXT: addi sp, sp, 80 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load ptr, ptr %va + %ptrint = ptrtoint ptr %argp.cur to iXLen + %1 = add iXLen %ptrint, 7 + %2 = and iXLen %1, -8 + %argp.cur.aligned = inttoptr iXLen %1 to ptr + %argp.next = getelementptr inbounds i8, ptr %argp.cur.aligned, i32 8 + store ptr %argp.next, ptr %va + %3 = inttoptr iXLen %2 to ptr + %4 = load double, ptr %3, align 8 + %5 = bitcast double %4 to i64 + call void @llvm.va_end(ptr %va) + ret i64 %5 +} + +; This test is slightly different than the SelectionDAG counterpart because +; narrowScalar and widenScalar for G_VAARG on types outside of [s32, sXLen] +; are not implemented yet. +define i64 @va2_va_arg(ptr %fmt, ...) nounwind { +; RV32-LABEL: va2_va_arg: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -48 +; RV32-NEXT: sw a1, 20(sp) +; RV32-NEXT: sw a2, 24(sp) +; RV32-NEXT: sw a3, 28(sp) +; RV32-NEXT: sw a4, 32(sp) +; RV32-NEXT: sw a5, 36(sp) +; RV32-NEXT: sw a6, 40(sp) +; RV32-NEXT: sw a7, 44(sp) +; RV32-NEXT: addi a0, sp, 20 +; RV32-NEXT: sw a0, 12(sp) +; RV32-NEXT: lw a0, 12(sp) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 12(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: li a1, 0 +; RV32-NEXT: addi sp, sp, 48 +; RV32-NEXT: ret +; +; RV64-LABEL: va2_va_arg: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -80 +; RV64-NEXT: sd a1, 24(sp) +; RV64-NEXT: sd a2, 32(sp) +; RV64-NEXT: sd a3, 40(sp) +; RV64-NEXT: sd a4, 48(sp) +; RV64-NEXT: sd a5, 56(sp) +; RV64-NEXT: sd a6, 64(sp) +; RV64-NEXT: sd a7, 72(sp) +; RV64-NEXT: addi a0, sp, 24 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: sd a1, 8(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: slli a0, a0, 32 +; RV64-NEXT: srli a0, a0, 32 +; RV64-NEXT: addi sp, sp, 80 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + call void @llvm.va_end(ptr %va) + %2 = zext i32 %1 to i64 + ret i64 %2 +} + +define void @va2_caller() nounwind { +; RV32-LABEL: va2_caller: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -16 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: li a1, 1 +; RV32-NEXT: call va2@plt +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 16 +; RV32-NEXT: ret +; +; RV64-LABEL: va2_caller: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -16 +; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: li a1, 1 +; RV64-NEXT: call va2@plt +; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 16 +; RV64-NEXT: ret + %1 = call i64 (ptr, ...) @va2(ptr undef, i32 1) + ret void +} + +; On RV32, Ensure a named 2*xlen argument is passed in a1 and a2, while the +; vararg double is passed in a4 and a5 (rather than a3 and a4) + +define i64 @va3(i32 %a, i64 %b, ...) nounwind { +; ILP32-LABEL: va3: +; ILP32: # %bb.0: +; ILP32-NEXT: addi sp, sp, -32 +; ILP32-NEXT: sw a3, 12(sp) +; ILP32-NEXT: sw a4, 16(sp) +; ILP32-NEXT: addi a0, sp, 12 +; ILP32-NEXT: sw a0, 4(sp) +; ILP32-NEXT: lw a0, 4(sp) +; ILP32-NEXT: sw a5, 20(sp) +; ILP32-NEXT: sw a6, 24(sp) +; ILP32-NEXT: sw a7, 28(sp) +; ILP32-NEXT: addi a3, a0, 7 +; ILP32-NEXT: andi a3, a3, -8 +; ILP32-NEXT: addi a0, a0, 15 +; ILP32-NEXT: sw a0, 4(sp) +; ILP32-NEXT: lw a4, 0(a3) +; ILP32-NEXT: lw a3, 4(a3) +; ILP32-NEXT: add a0, a1, a4 +; ILP32-NEXT: sltu a1, a0, a4 +; ILP32-NEXT: add a2, a2, a3 +; ILP32-NEXT: add a1, a2, a1 +; ILP32-NEXT: addi sp, sp, 32 +; ILP32-NEXT: ret +; +; RV32D-ILP32-LABEL: va3: +; RV32D-ILP32: # %bb.0: +; RV32D-ILP32-NEXT: addi sp, sp, -48 +; RV32D-ILP32-NEXT: sw a3, 28(sp) +; RV32D-ILP32-NEXT: sw a4, 32(sp) +; RV32D-ILP32-NEXT: addi a0, sp, 28 +; RV32D-ILP32-NEXT: sw a0, 20(sp) +; RV32D-ILP32-NEXT: lw a0, 20(sp) +; RV32D-ILP32-NEXT: sw a5, 36(sp) +; RV32D-ILP32-NEXT: sw a6, 40(sp) +; RV32D-ILP32-NEXT: sw a7, 44(sp) +; RV32D-ILP32-NEXT: addi a3, a0, 7 +; RV32D-ILP32-NEXT: andi a3, a3, -8 +; RV32D-ILP32-NEXT: fld fa5, 0(a3) +; RV32D-ILP32-NEXT: addi a0, a0, 15 +; RV32D-ILP32-NEXT: sw a0, 20(sp) +; RV32D-ILP32-NEXT: fsd fa5, 8(sp) +; RV32D-ILP32-NEXT: lw a3, 8(sp) +; RV32D-ILP32-NEXT: lw a4, 12(sp) +; RV32D-ILP32-NEXT: add a0, a1, a3 +; RV32D-ILP32-NEXT: sltu a1, a0, a3 +; RV32D-ILP32-NEXT: add a2, a2, a4 +; RV32D-ILP32-NEXT: add a1, a2, a1 +; RV32D-ILP32-NEXT: addi sp, sp, 48 +; RV32D-ILP32-NEXT: ret +; +; RV32D-ILP32F-LABEL: va3: +; RV32D-ILP32F: # %bb.0: +; RV32D-ILP32F-NEXT: addi sp, sp, -48 +; RV32D-ILP32F-NEXT: sw a3, 28(sp) +; RV32D-ILP32F-NEXT: sw a4, 32(sp) +; RV32D-ILP32F-NEXT: addi a0, sp, 28 +; RV32D-ILP32F-NEXT: sw a0, 20(sp) +; RV32D-ILP32F-NEXT: lw a0, 20(sp) +; RV32D-ILP32F-NEXT: sw a5, 36(sp) +; RV32D-ILP32F-NEXT: sw a6, 40(sp) +; RV32D-ILP32F-NEXT: sw a7, 44(sp) +; RV32D-ILP32F-NEXT: addi a3, a0, 7 +; RV32D-ILP32F-NEXT: andi a3, a3, -8 +; RV32D-ILP32F-NEXT: fld fa5, 0(a3) +; RV32D-ILP32F-NEXT: addi a0, a0, 15 +; RV32D-ILP32F-NEXT: sw a0, 20(sp) +; RV32D-ILP32F-NEXT: fsd fa5, 8(sp) +; RV32D-ILP32F-NEXT: lw a3, 8(sp) +; RV32D-ILP32F-NEXT: lw a4, 12(sp) +; RV32D-ILP32F-NEXT: add a0, a1, a3 +; RV32D-ILP32F-NEXT: sltu a1, a0, a3 +; RV32D-ILP32F-NEXT: add a2, a2, a4 +; RV32D-ILP32F-NEXT: add a1, a2, a1 +; RV32D-ILP32F-NEXT: addi sp, sp, 48 +; RV32D-ILP32F-NEXT: ret +; +; RV32D-ILP32D-LABEL: va3: +; RV32D-ILP32D: # %bb.0: +; RV32D-ILP32D-NEXT: addi sp, sp, -48 +; RV32D-ILP32D-NEXT: sw a3, 28(sp) +; RV32D-ILP32D-NEXT: sw a4, 32(sp) +; RV32D-ILP32D-NEXT: addi a0, sp, 28 +; RV32D-ILP32D-NEXT: sw a0, 20(sp) +; RV32D-ILP32D-NEXT: lw a0, 20(sp) +; RV32D-ILP32D-NEXT: sw a5, 36(sp) +; RV32D-ILP32D-NEXT: sw a6, 40(sp) +; RV32D-ILP32D-NEXT: sw a7, 44(sp) +; RV32D-ILP32D-NEXT: addi a3, a0, 7 +; RV32D-ILP32D-NEXT: andi a3, a3, -8 +; RV32D-ILP32D-NEXT: fld fa5, 0(a3) +; RV32D-ILP32D-NEXT: addi a0, a0, 15 +; RV32D-ILP32D-NEXT: sw a0, 20(sp) +; RV32D-ILP32D-NEXT: fsd fa5, 8(sp) +; RV32D-ILP32D-NEXT: lw a3, 8(sp) +; RV32D-ILP32D-NEXT: lw a4, 12(sp) +; RV32D-ILP32D-NEXT: add a0, a1, a3 +; RV32D-ILP32D-NEXT: sltu a1, a0, a3 +; RV32D-ILP32D-NEXT: add a2, a2, a4 +; RV32D-ILP32D-NEXT: add a1, a2, a1 +; RV32D-ILP32D-NEXT: addi sp, sp, 48 +; RV32D-ILP32D-NEXT: ret +; +; RV64-LABEL: va3: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -64 +; RV64-NEXT: sd a2, 16(sp) +; RV64-NEXT: sd a3, 24(sp) +; RV64-NEXT: sd a4, 32(sp) +; RV64-NEXT: addi a0, sp, 16 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: sd a5, 40(sp) +; RV64-NEXT: sd a6, 48(sp) +; RV64-NEXT: sd a7, 56(sp) +; RV64-NEXT: addi a2, a0, 7 +; RV64-NEXT: andi a2, a2, -8 +; RV64-NEXT: addi a0, a0, 15 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 0(a2) +; RV64-NEXT: add a0, a1, a0 +; RV64-NEXT: addi sp, sp, 64 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load ptr, ptr %va + %ptrint = ptrtoint ptr %argp.cur to iXLen + %1 = add iXLen %ptrint, 7 + %2 = and iXLen %1, -8 + %argp.cur.aligned = inttoptr iXLen %1 to ptr + %argp.next = getelementptr inbounds i8, ptr %argp.cur.aligned, i32 8 + store ptr %argp.next, ptr %va + %3 = inttoptr iXLen %2 to ptr + %4 = load double, ptr %3, align 8 + call void @llvm.va_end(ptr %va) + %5 = bitcast double %4 to i64 + %6 = add i64 %b, %5 + ret i64 %6 +} + +; This test is slightly different than the SelectionDAG counterpart because +; narrowScalar and widenScalar for G_VAARG on types outside of [s32, sXLen] +; are not implemented yet. +define i64 @va3_va_arg(i32 %a, i64 %b, ...) nounwind { +; RV32-LABEL: va3_va_arg: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -32 +; RV32-NEXT: sw a3, 12(sp) +; RV32-NEXT: sw a4, 16(sp) +; RV32-NEXT: sw a5, 20(sp) +; RV32-NEXT: sw a6, 24(sp) +; RV32-NEXT: sw a7, 28(sp) +; RV32-NEXT: addi a0, sp, 12 +; RV32-NEXT: sw a0, 4(sp) +; RV32-NEXT: lw a0, 4(sp) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a3, a0, 4 +; RV32-NEXT: sw a3, 4(sp) +; RV32-NEXT: lw a3, 0(a0) +; RV32-NEXT: add a0, a1, a3 +; RV32-NEXT: sltu a1, a0, a3 +; RV32-NEXT: add a1, a2, a1 +; RV32-NEXT: addi sp, sp, 32 +; RV32-NEXT: ret +; +; RV64-LABEL: va3_va_arg: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -64 +; RV64-NEXT: sd a2, 16(sp) +; RV64-NEXT: sd a3, 24(sp) +; RV64-NEXT: sd a4, 32(sp) +; RV64-NEXT: sd a5, 40(sp) +; RV64-NEXT: sd a6, 48(sp) +; RV64-NEXT: sd a7, 56(sp) +; RV64-NEXT: addi a0, sp, 16 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a2, a0, 4 +; RV64-NEXT: sd a2, 8(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: slli a0, a0, 32 +; RV64-NEXT: srli a0, a0, 32 +; RV64-NEXT: add a0, a1, a0 +; RV64-NEXT: addi sp, sp, 64 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + call void @llvm.va_end(ptr %va) + %2 = zext i32 %1 to i64 + %3 = add i64 %b, %2 + ret i64 %3 +} + +define void @va3_caller() nounwind { +; RV32-LABEL: va3_caller: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -16 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: lui a0, 5 +; RV32-NEXT: addi a3, a0, -480 +; RV32-NEXT: li a0, 2 +; RV32-NEXT: li a1, 1111 +; RV32-NEXT: li a2, 0 +; RV32-NEXT: call va3@plt +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 16 +; RV32-NEXT: ret +; +; RV64-LABEL: va3_caller: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -16 +; RV64-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64-NEXT: lui a0, 5 +; RV64-NEXT: addiw a2, a0, -480 +; RV64-NEXT: li a0, 2 +; RV64-NEXT: li a1, 1111 +; RV64-NEXT: call va3@plt +; RV64-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 16 +; RV64-NEXT: ret + %1 = call i64 (i32, i64, ...) @va3(i32 2, i64 1111, i32 20000) + ret void +} + +declare void @llvm.va_copy(ptr, ptr) + +define i32 @va4_va_copy(i32 %argno, ...) nounwind { +; RV32-LABEL: va4_va_copy: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -48 +; RV32-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32-NEXT: sw s0, 8(sp) # 4-byte Folded Spill +; RV32-NEXT: sw a1, 20(sp) +; RV32-NEXT: sw a2, 24(sp) +; RV32-NEXT: sw a3, 28(sp) +; RV32-NEXT: sw a4, 32(sp) +; RV32-NEXT: sw a5, 36(sp) +; RV32-NEXT: sw a6, 40(sp) +; RV32-NEXT: sw a7, 44(sp) +; RV32-NEXT: addi a0, sp, 20 +; RV32-NEXT: sw a0, 4(sp) +; RV32-NEXT: lw a0, 4(sp) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 4(sp) +; RV32-NEXT: lw a1, 4(sp) +; RV32-NEXT: mv a2, sp +; RV32-NEXT: lw s0, 0(a0) +; RV32-NEXT: sw a2, 0(a1) +; RV32-NEXT: lw a0, 0(sp) +; RV32-NEXT: call notdead@plt +; RV32-NEXT: lw a0, 4(sp) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 4(sp) +; RV32-NEXT: lw a1, 4(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: addi a1, a1, 3 +; RV32-NEXT: andi a1, a1, -4 +; RV32-NEXT: addi a2, a1, 4 +; RV32-NEXT: sw a2, 4(sp) +; RV32-NEXT: lw a2, 4(sp) +; RV32-NEXT: lw a1, 0(a1) +; RV32-NEXT: addi a2, a2, 3 +; RV32-NEXT: andi a2, a2, -4 +; RV32-NEXT: addi a3, a2, 4 +; RV32-NEXT: sw a3, 4(sp) +; RV32-NEXT: lw a2, 0(a2) +; RV32-NEXT: add a0, a0, s0 +; RV32-NEXT: add a1, a1, a2 +; RV32-NEXT: add a0, a0, a1 +; RV32-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32-NEXT: lw s0, 8(sp) # 4-byte Folded Reload +; RV32-NEXT: addi sp, sp, 48 +; RV32-NEXT: ret +; +; RV64-LABEL: va4_va_copy: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -96 +; RV64-NEXT: sd ra, 24(sp) # 8-byte Folded Spill +; RV64-NEXT: sd s0, 16(sp) # 8-byte Folded Spill +; RV64-NEXT: sd a1, 40(sp) +; RV64-NEXT: sd a2, 48(sp) +; RV64-NEXT: sd a3, 56(sp) +; RV64-NEXT: sd a4, 64(sp) +; RV64-NEXT: sd a5, 72(sp) +; RV64-NEXT: sd a6, 80(sp) +; RV64-NEXT: sd a7, 88(sp) +; RV64-NEXT: addi a0, sp, 40 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: sd a1, 8(sp) +; RV64-NEXT: ld a1, 8(sp) +; RV64-NEXT: mv a2, sp +; RV64-NEXT: lw s0, 0(a0) +; RV64-NEXT: sd a2, 0(a1) +; RV64-NEXT: lw a0, 4(sp) +; RV64-NEXT: lwu a1, 0(sp) +; RV64-NEXT: slli a0, a0, 32 +; RV64-NEXT: or a0, a0, a1 +; RV64-NEXT: call notdead@plt +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: sd a1, 8(sp) +; RV64-NEXT: ld a1, 8(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: addi a1, a1, 3 +; RV64-NEXT: andi a1, a1, -4 +; RV64-NEXT: addi a2, a1, 4 +; RV64-NEXT: sd a2, 8(sp) +; RV64-NEXT: ld a2, 8(sp) +; RV64-NEXT: lw a1, 0(a1) +; RV64-NEXT: addi a2, a2, 3 +; RV64-NEXT: andi a2, a2, -4 +; RV64-NEXT: addi a3, a2, 4 +; RV64-NEXT: sd a3, 8(sp) +; RV64-NEXT: lw a2, 0(a2) +; RV64-NEXT: add a0, a0, s0 +; RV64-NEXT: add a1, a1, a2 +; RV64-NEXT: addw a0, a0, a1 +; RV64-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; RV64-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; RV64-NEXT: addi sp, sp, 96 +; RV64-NEXT: ret + %vargs = alloca ptr + %wargs = alloca ptr + call void @llvm.va_start(ptr %vargs) + %1 = va_arg ptr %vargs, i32 + call void @llvm.va_copy(ptr %wargs, ptr %vargs) + %2 = load ptr, ptr %wargs, align 4 + call void @notdead(ptr %2) + %3 = va_arg ptr %vargs, i32 + %4 = va_arg ptr %vargs, i32 + %5 = va_arg ptr %vargs, i32 + call void @llvm.va_end(ptr %vargs) + call void @llvm.va_end(ptr %wargs) + %add1 = add i32 %3, %1 + %add2 = add i32 %add1, %4 + %add3 = add i32 %add2, %5 + ret i32 %add3 +} + +; The va5_aligned_stack_callee and caller function are ommitted from this file +; since they were not included in the IR lowering test when vararg calls were +; initially added. + +; A function with no fixed arguments is not valid C, but can be +; specified in LLVM IR. We must ensure the vararg save area is +; still set up correctly. + +define i32 @va6_no_fixed_args(...) nounwind { +; RV32-LABEL: va6_no_fixed_args: +; RV32: # %bb.0: +; RV32-NEXT: addi sp, sp, -48 +; RV32-NEXT: sw a0, 16(sp) +; RV32-NEXT: sw a1, 20(sp) +; RV32-NEXT: sw a2, 24(sp) +; RV32-NEXT: sw a3, 28(sp) +; RV32-NEXT: sw a4, 32(sp) +; RV32-NEXT: sw a5, 36(sp) +; RV32-NEXT: sw a6, 40(sp) +; RV32-NEXT: sw a7, 44(sp) +; RV32-NEXT: addi a0, sp, 16 +; RV32-NEXT: sw a0, 12(sp) +; RV32-NEXT: lw a0, 12(sp) +; RV32-NEXT: addi a0, a0, 3 +; RV32-NEXT: andi a0, a0, -4 +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 12(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: addi sp, sp, 48 +; RV32-NEXT: ret +; +; RV64-LABEL: va6_no_fixed_args: +; RV64: # %bb.0: +; RV64-NEXT: addi sp, sp, -80 +; RV64-NEXT: sd a0, 16(sp) +; RV64-NEXT: sd a1, 24(sp) +; RV64-NEXT: sd a2, 32(sp) +; RV64-NEXT: sd a3, 40(sp) +; RV64-NEXT: sd a4, 48(sp) +; RV64-NEXT: sd a5, 56(sp) +; RV64-NEXT: sd a6, 64(sp) +; RV64-NEXT: sd a7, 72(sp) +; RV64-NEXT: addi a0, sp, 16 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: ld a0, 8(sp) +; RV64-NEXT: addi a0, a0, 3 +; RV64-NEXT: andi a0, a0, -4 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: sd a1, 8(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: addi sp, sp, 80 +; RV64-NEXT: ret + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %1 = va_arg ptr %va, i32 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + +; TODO: improve constant materialization of stack addresses + +define i32 @va_large_stack(ptr %fmt, ...) { +; RV32-LABEL: va_large_stack: +; RV32: # %bb.0: +; RV32-NEXT: lui a0, 24414 +; RV32-NEXT: addi a0, a0, 304 +; RV32-NEXT: sub sp, sp, a0 +; RV32-NEXT: .cfi_def_cfa_offset 100000048 +; RV32-NEXT: lui a0, 24414 +; RV32-NEXT: add a0, sp, a0 +; RV32-NEXT: sw a1, 276(a0) +; RV32-NEXT: lui a0, 24414 +; RV32-NEXT: add a0, sp, a0 +; RV32-NEXT: sw a2, 280(a0) +; RV32-NEXT: lui a0, 24414 +; RV32-NEXT: add a0, sp, a0 +; RV32-NEXT: sw a3, 284(a0) +; RV32-NEXT: lui a0, 24414 +; RV32-NEXT: add a0, sp, a0 +; RV32-NEXT: sw a4, 288(a0) +; RV32-NEXT: lui a0, 24414 +; RV32-NEXT: addi a0, a0, 276 +; RV32-NEXT: add a0, sp, a0 +; RV32-NEXT: sw a0, 12(sp) +; RV32-NEXT: lw a0, 12(sp) +; RV32-NEXT: lui a1, 24414 +; RV32-NEXT: add a1, sp, a1 +; RV32-NEXT: sw a5, 292(a1) +; RV32-NEXT: lui a1, 24414 +; RV32-NEXT: add a1, sp, a1 +; RV32-NEXT: sw a6, 296(a1) +; RV32-NEXT: lui a1, 24414 +; RV32-NEXT: add a1, sp, a1 +; RV32-NEXT: sw a7, 300(a1) +; RV32-NEXT: addi a1, a0, 4 +; RV32-NEXT: sw a1, 12(sp) +; RV32-NEXT: lw a0, 0(a0) +; RV32-NEXT: lui a1, 24414 +; RV32-NEXT: addi a1, a1, 304 +; RV32-NEXT: add sp, sp, a1 +; RV32-NEXT: ret +; +; RV64-LABEL: va_large_stack: +; RV64: # %bb.0: +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: addiw a0, a0, 336 +; RV64-NEXT: sub sp, sp, a0 +; RV64-NEXT: .cfi_def_cfa_offset 100000080 +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: add a0, sp, a0 +; RV64-NEXT: sd a1, 280(a0) +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: add a0, sp, a0 +; RV64-NEXT: sd a2, 288(a0) +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: add a0, sp, a0 +; RV64-NEXT: sd a3, 296(a0) +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: add a0, sp, a0 +; RV64-NEXT: sd a4, 304(a0) +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: add a0, sp, a0 +; RV64-NEXT: sd a5, 312(a0) +; RV64-NEXT: lui a0, 24414 +; RV64-NEXT: addiw a0, a0, 280 +; RV64-NEXT: add a0, sp, a0 +; RV64-NEXT: sd a0, 8(sp) +; RV64-NEXT: lw a0, 12(sp) +; RV64-NEXT: lwu a1, 8(sp) +; RV64-NEXT: lui a2, 24414 +; RV64-NEXT: add a2, sp, a2 +; RV64-NEXT: sd a6, 320(a2) +; RV64-NEXT: lui a2, 24414 +; RV64-NEXT: add a2, sp, a2 +; RV64-NEXT: sd a7, 328(a2) +; RV64-NEXT: slli a0, a0, 32 +; RV64-NEXT: or a0, a0, a1 +; RV64-NEXT: addi a1, a0, 4 +; RV64-NEXT: srli a2, a1, 32 +; RV64-NEXT: sw a1, 8(sp) +; RV64-NEXT: sw a2, 12(sp) +; RV64-NEXT: lw a0, 0(a0) +; RV64-NEXT: lui a1, 24414 +; RV64-NEXT: addiw a1, a1, 336 +; RV64-NEXT: add sp, sp, a1 +; RV64-NEXT: ret + %large = alloca [ 100000000 x i8 ] + %va = alloca ptr + call void @llvm.va_start(ptr %va) + %argp.cur = load ptr, ptr %va, align 4 + %argp.next = getelementptr inbounds i8, ptr %argp.cur, i32 4 + store ptr %argp.next, ptr %va, align 4 + %1 = load i32, ptr %argp.cur, align 4 + call void @llvm.va_end(ptr %va) + ret i32 %1 +} + + + diff --git a/llvm/test/CodeGen/RISCV/llvm.frexp.ll b/llvm/test/CodeGen/RISCV/llvm.frexp.ll index f8a8cfc60f14..94b9444dfaf8 100644 --- a/llvm/test/CodeGen/RISCV/llvm.frexp.ll +++ b/llvm/test/CodeGen/RISCV/llvm.frexp.ll @@ -18,24 +18,279 @@ ; TODO: FIXMEs are copied blindly across from the X86 version of this test. -; FIXME -; define { half, i32 } @test_frexp_f16_i32(half %a) nounwind { -; %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) -; ret { half, i32 } %result -; } +define { half, i32 } @test_frexp_f16_i32(half %a) nounwind { +; RV32IFD-LABEL: test_frexp_f16_i32: +; RV32IFD: # %bb.0: +; RV32IFD-NEXT: addi sp, sp, -16 +; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: call __extendhfsf2@plt +; RV32IFD-NEXT: addi a0, sp, 8 +; RV32IFD-NEXT: call frexpf@plt +; RV32IFD-NEXT: call __truncsfhf2@plt +; RV32IFD-NEXT: fmv.x.w a1, fa0 +; RV32IFD-NEXT: lw a0, 8(sp) +; RV32IFD-NEXT: lui a2, 1048560 +; RV32IFD-NEXT: or a1, a1, a2 +; RV32IFD-NEXT: fmv.w.x fa0, a1 +; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: ret +; +; RV64IFD-LABEL: test_frexp_f16_i32: +; RV64IFD: # %bb.0: +; RV64IFD-NEXT: addi sp, sp, -16 +; RV64IFD-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64IFD-NEXT: call __extendhfsf2@plt +; RV64IFD-NEXT: mv a0, sp +; RV64IFD-NEXT: call frexpf@plt +; RV64IFD-NEXT: call __truncsfhf2@plt +; RV64IFD-NEXT: fmv.x.w a1, fa0 +; RV64IFD-NEXT: ld a0, 0(sp) +; RV64IFD-NEXT: lui a2, 1048560 +; RV64IFD-NEXT: or a1, a1, a2 +; RV64IFD-NEXT: fmv.w.x fa0, a1 +; RV64IFD-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64IFD-NEXT: addi sp, sp, 16 +; RV64IFD-NEXT: ret +; +; RV32IZFINXZDINX-LABEL: test_frexp_f16_i32: +; RV32IZFINXZDINX: # %bb.0: +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: call __extendhfsf2@plt +; RV32IZFINXZDINX-NEXT: addi a1, sp, 8 +; RV32IZFINXZDINX-NEXT: call frexpf@plt +; RV32IZFINXZDINX-NEXT: call __truncsfhf2@plt +; RV32IZFINXZDINX-NEXT: lw a1, 8(sp) +; RV32IZFINXZDINX-NEXT: lui a2, 1048560 +; RV32IZFINXZDINX-NEXT: or a0, a0, a2 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: ret +; +; RV64IZFINXZDINX-LABEL: test_frexp_f16_i32: +; RV64IZFINXZDINX: # %bb.0: +; RV64IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV64IZFINXZDINX-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64IZFINXZDINX-NEXT: call __extendhfsf2@plt +; RV64IZFINXZDINX-NEXT: mv a1, sp +; RV64IZFINXZDINX-NEXT: call frexpf@plt +; RV64IZFINXZDINX-NEXT: call __truncsfhf2@plt +; RV64IZFINXZDINX-NEXT: ld a1, 0(sp) +; RV64IZFINXZDINX-NEXT: lui a2, 1048560 +; RV64IZFINXZDINX-NEXT: or a0, a0, a2 +; RV64IZFINXZDINX-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV64IZFINXZDINX-NEXT: ret +; +; RV32I-LABEL: test_frexp_f16_i32: +; RV32I: # %bb.0: +; RV32I-NEXT: addi sp, sp, -16 +; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32I-NEXT: slli a0, a0, 16 +; RV32I-NEXT: srli a0, a0, 16 +; RV32I-NEXT: call __extendhfsf2@plt +; RV32I-NEXT: addi a1, sp, 8 +; RV32I-NEXT: call frexpf@plt +; RV32I-NEXT: call __truncsfhf2@plt +; RV32I-NEXT: lw a1, 8(sp) +; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32I-NEXT: addi sp, sp, 16 +; RV32I-NEXT: ret +; +; RV64I-LABEL: test_frexp_f16_i32: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -16 +; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: slli a0, a0, 48 +; RV64I-NEXT: srli a0, a0, 48 +; RV64I-NEXT: call __extendhfsf2@plt +; RV64I-NEXT: addi a1, sp, 4 +; RV64I-NEXT: call frexpf@plt +; RV64I-NEXT: call __truncsfhf2@plt +; RV64I-NEXT: lw a1, 4(sp) +; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: addi sp, sp, 16 +; RV64I-NEXT: ret + %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + ret { half, i32 } %result +} -; define half @test_frexp_f16_i32_only_use_fract(half %a) nounwind { -; %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) -; %result.0 = extractvalue { half, i32 } %result, 0 -; ret half %result.0 -; } +define half @test_frexp_f16_i32_only_use_fract(half %a) nounwind { +; RV32IFD-LABEL: test_frexp_f16_i32_only_use_fract: +; RV32IFD: # %bb.0: +; RV32IFD-NEXT: addi sp, sp, -16 +; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: call __extendhfsf2@plt +; RV32IFD-NEXT: addi a0, sp, 8 +; RV32IFD-NEXT: call frexpf@plt +; RV32IFD-NEXT: call __truncsfhf2@plt +; RV32IFD-NEXT: fmv.x.w a0, fa0 +; RV32IFD-NEXT: lui a1, 1048560 +; RV32IFD-NEXT: or a0, a0, a1 +; RV32IFD-NEXT: fmv.w.x fa0, a0 +; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: ret +; +; RV64IFD-LABEL: test_frexp_f16_i32_only_use_fract: +; RV64IFD: # %bb.0: +; RV64IFD-NEXT: addi sp, sp, -16 +; RV64IFD-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64IFD-NEXT: call __extendhfsf2@plt +; RV64IFD-NEXT: mv a0, sp +; RV64IFD-NEXT: call frexpf@plt +; RV64IFD-NEXT: call __truncsfhf2@plt +; RV64IFD-NEXT: fmv.x.w a0, fa0 +; RV64IFD-NEXT: lui a1, 1048560 +; RV64IFD-NEXT: or a0, a0, a1 +; RV64IFD-NEXT: fmv.w.x fa0, a0 +; RV64IFD-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64IFD-NEXT: addi sp, sp, 16 +; RV64IFD-NEXT: ret +; +; RV32IZFINXZDINX-LABEL: test_frexp_f16_i32_only_use_fract: +; RV32IZFINXZDINX: # %bb.0: +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: call __extendhfsf2@plt +; RV32IZFINXZDINX-NEXT: addi a1, sp, 8 +; RV32IZFINXZDINX-NEXT: call frexpf@plt +; RV32IZFINXZDINX-NEXT: call __truncsfhf2@plt +; RV32IZFINXZDINX-NEXT: lui a1, 1048560 +; RV32IZFINXZDINX-NEXT: or a0, a0, a1 +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: ret +; +; RV64IZFINXZDINX-LABEL: test_frexp_f16_i32_only_use_fract: +; RV64IZFINXZDINX: # %bb.0: +; RV64IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV64IZFINXZDINX-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64IZFINXZDINX-NEXT: call __extendhfsf2@plt +; RV64IZFINXZDINX-NEXT: mv a1, sp +; RV64IZFINXZDINX-NEXT: call frexpf@plt +; RV64IZFINXZDINX-NEXT: call __truncsfhf2@plt +; RV64IZFINXZDINX-NEXT: lui a1, 1048560 +; RV64IZFINXZDINX-NEXT: or a0, a0, a1 +; RV64IZFINXZDINX-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV64IZFINXZDINX-NEXT: ret +; +; RV32I-LABEL: test_frexp_f16_i32_only_use_fract: +; RV32I: # %bb.0: +; RV32I-NEXT: addi sp, sp, -16 +; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32I-NEXT: slli a0, a0, 16 +; RV32I-NEXT: srli a0, a0, 16 +; RV32I-NEXT: call __extendhfsf2@plt +; RV32I-NEXT: addi a1, sp, 8 +; RV32I-NEXT: call frexpf@plt +; RV32I-NEXT: call __truncsfhf2@plt +; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32I-NEXT: addi sp, sp, 16 +; RV32I-NEXT: ret +; +; RV64I-LABEL: test_frexp_f16_i32_only_use_fract: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -16 +; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: slli a0, a0, 48 +; RV64I-NEXT: srli a0, a0, 48 +; RV64I-NEXT: call __extendhfsf2@plt +; RV64I-NEXT: addi a1, sp, 4 +; RV64I-NEXT: call frexpf@plt +; RV64I-NEXT: call __truncsfhf2@plt +; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: addi sp, sp, 16 +; RV64I-NEXT: ret + %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + %result.0 = extractvalue { half, i32 } %result, 0 + ret half %result.0 +} -; define i32 @test_frexp_f16_i32_only_use_exp(half %a) nounwind { -; %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) -; %result.0 = extractvalue { half, i32 } %result, 1 -; ret i32 %result.0 -; } +define i32 @test_frexp_f16_i32_only_use_exp(half %a) nounwind { +; RV32IFD-LABEL: test_frexp_f16_i32_only_use_exp: +; RV32IFD: # %bb.0: +; RV32IFD-NEXT: addi sp, sp, -16 +; RV32IFD-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IFD-NEXT: call __extendhfsf2@plt +; RV32IFD-NEXT: addi a0, sp, 8 +; RV32IFD-NEXT: call frexpf@plt +; RV32IFD-NEXT: lw a0, 8(sp) +; RV32IFD-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IFD-NEXT: addi sp, sp, 16 +; RV32IFD-NEXT: ret +; +; RV64IFD-LABEL: test_frexp_f16_i32_only_use_exp: +; RV64IFD: # %bb.0: +; RV64IFD-NEXT: addi sp, sp, -16 +; RV64IFD-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64IFD-NEXT: call __extendhfsf2@plt +; RV64IFD-NEXT: mv a0, sp +; RV64IFD-NEXT: call frexpf@plt +; RV64IFD-NEXT: ld a0, 0(sp) +; RV64IFD-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64IFD-NEXT: addi sp, sp, 16 +; RV64IFD-NEXT: ret +; +; RV32IZFINXZDINX-LABEL: test_frexp_f16_i32_only_use_exp: +; RV32IZFINXZDINX: # %bb.0: +; RV32IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV32IZFINXZDINX-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32IZFINXZDINX-NEXT: call __extendhfsf2@plt +; RV32IZFINXZDINX-NEXT: addi a1, sp, 8 +; RV32IZFINXZDINX-NEXT: call frexpf@plt +; RV32IZFINXZDINX-NEXT: lw a0, 8(sp) +; RV32IZFINXZDINX-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV32IZFINXZDINX-NEXT: ret +; +; RV64IZFINXZDINX-LABEL: test_frexp_f16_i32_only_use_exp: +; RV64IZFINXZDINX: # %bb.0: +; RV64IZFINXZDINX-NEXT: addi sp, sp, -16 +; RV64IZFINXZDINX-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64IZFINXZDINX-NEXT: call __extendhfsf2@plt +; RV64IZFINXZDINX-NEXT: mv a1, sp +; RV64IZFINXZDINX-NEXT: call frexpf@plt +; RV64IZFINXZDINX-NEXT: ld a0, 0(sp) +; RV64IZFINXZDINX-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64IZFINXZDINX-NEXT: addi sp, sp, 16 +; RV64IZFINXZDINX-NEXT: ret +; +; RV32I-LABEL: test_frexp_f16_i32_only_use_exp: +; RV32I: # %bb.0: +; RV32I-NEXT: addi sp, sp, -16 +; RV32I-NEXT: sw ra, 12(sp) # 4-byte Folded Spill +; RV32I-NEXT: slli a0, a0, 16 +; RV32I-NEXT: srli a0, a0, 16 +; RV32I-NEXT: call __extendhfsf2@plt +; RV32I-NEXT: addi a1, sp, 8 +; RV32I-NEXT: call frexpf@plt +; RV32I-NEXT: lw a0, 8(sp) +; RV32I-NEXT: lw ra, 12(sp) # 4-byte Folded Reload +; RV32I-NEXT: addi sp, sp, 16 +; RV32I-NEXT: ret +; +; RV64I-LABEL: test_frexp_f16_i32_only_use_exp: +; RV64I: # %bb.0: +; RV64I-NEXT: addi sp, sp, -16 +; RV64I-NEXT: sd ra, 8(sp) # 8-byte Folded Spill +; RV64I-NEXT: slli a0, a0, 48 +; RV64I-NEXT: srli a0, a0, 48 +; RV64I-NEXT: call __extendhfsf2@plt +; RV64I-NEXT: addi a1, sp, 4 +; RV64I-NEXT: call frexpf@plt +; RV64I-NEXT: lw a0, 4(sp) +; RV64I-NEXT: ld ra, 8(sp) # 8-byte Folded Reload +; RV64I-NEXT: addi sp, sp, 16 +; RV64I-NEXT: ret + %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + %result.0 = extractvalue { half, i32 } %result, 1 + ret i32 %result.0 +} +; FIXME ; define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) nounwind { ; %result = call { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half> %a) ; ret { <2 x half>, <2 x i32> } %result diff --git a/llvm/test/CodeGen/RISCV/macro-fusions-veyron-v1.mir b/llvm/test/CodeGen/RISCV/macro-fusions-veyron-v1.mir new file mode 100644 index 000000000000..6d1e92e997b3 --- /dev/null +++ b/llvm/test/CodeGen/RISCV/macro-fusions-veyron-v1.mir @@ -0,0 +1,159 @@ +# REQUIRES: asserts +# RUN: llc -mtriple=riscv64-linux-gnu -mcpu=veyron-v1 -x=mir < %s \ +# RUN: -debug-only=machine-scheduler -start-before=machine-scheduler 2>&1 \ +# RUN: -mattr=+lui-addi-fusion,+auipc-addi-fusion,+shifted-zext-fusion,+ld-add-fusion \ +# RUN: | FileCheck %s + +# CHECK: lui_addi:%bb.0 +# CHECK: Macro fuse: {{.*}}LUI - ADDI +--- +name: lui_addi +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = LUI 1 + %3:gpr = XORI %1, 2 + %4:gpr = ADDI %2, 3 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: auipc_addi +# CHECK: Macro fuse: {{.*}}AUIPC - ADDI +--- +name: auipc_addi +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = AUIPC 1 + %3:gpr = XORI %1, 2 + %4:gpr = ADDI %2, 3 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: slli_srli +# CHECK: Macro fuse: {{.*}}SLLI - SRLI +--- +name: slli_srli +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = SLLI %1, 32 + %3:gpr = XORI %1, 3 + %4:gpr = SRLI %2, 4 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: slli_srli_48 +# CHECK: Macro fuse: {{.*}}SLLI - SRLI +--- +name: slli_srli_48 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = SLLI %1, 48 + %3:gpr = XORI %1, 3 + %4:gpr = SRLI %2, 48 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: slli_srli_no_fusion_0 +# CHECK-NOT: Macro fuse: {{.*}}SLLI - SRLI +--- +name: slli_srli_no_fusion_0 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = SLLI %1, 32 + %3:gpr = XORI %1, 3 + %4:gpr = SRLI %2, 33 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: slli_srli_no_fusion_1 +# CHECK-NOT: Macro fuse: {{.*}}SLLI - SRLI +--- +name: slli_srli_no_fusion_1 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = SLLI %1, 48 + %3:gpr = XORI %1, 3 + %4:gpr = SRLI %2, 4 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: slli_srli_no_fusion_2 +# CHECK-NOT: Macro fuse: {{.*}}SLLI - SRLI +--- +name: slli_srli_no_fusion_2 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = SLLI %1, 31 + %3:gpr = XORI %1, 3 + %4:gpr = SRLI %2, 4 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: slli_srli_no_fusion_3 +# CHECK-NOT: Macro fuse: {{.*}}SLLI - SRLI +--- +name: slli_srli_no_fusion_3 +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10 + %1:gpr = COPY $x10 + %2:gpr = SLLI %1, 31 + %3:gpr = XORI %1, 3 + %4:gpr = SRLI %2, 48 + $x10 = COPY %3 + $x11 = COPY %4 + PseudoRET +... + +# CHECK: ld_add +# CHECK: Macro fuse: {{.*}}ADD - LD +--- +name: ld_add +tracksRegLiveness: true +body: | + bb.0.entry: + liveins: $x10, $x11 + %1:gpr = COPY $x10 + %2:gpr = COPY $x11 + %3:gpr = ADD %1, %2 + %4:gpr = XORI %2, 3 + %5:gpr = LD %3, 0 + $x10 = COPY %4 + $x11 = COPY %5 + PseudoRET +... diff --git a/llvm/test/CodeGen/RISCV/relax-per-target-feature.ll b/llvm/test/CodeGen/RISCV/relax-per-target-feature.ll new file mode 100644 index 000000000000..7e94965cc59b --- /dev/null +++ b/llvm/test/CodeGen/RISCV/relax-per-target-feature.ll @@ -0,0 +1,34 @@ +; RUN: llc -filetype=obj -mtriple=riscv64 %s -o %t +; RUN: llvm-objdump -dr -M no-aliases --mattr=+c %t | FileCheck %s + +;; Functions may have more features than the base triple; code generation and +;; instruction selection may be performed based on this information. This test +;; makes sure that the MC layer uses the target-features of the function. + +declare dso_local i32 @ext(i32) + +; CHECK-LABEL: : +; CHECK-NEXT: c.li a0, 31 +; CHECK-NEXT: auipc t1, 0 +; CHECK-NEXT: R_RISCV_CALL_PLT ext +; CHECK-NEXT: R_RISCV_RELAX *ABS* +; CHECK-NEXT: jalr zero, 0(t1) +define dso_local i32 @f() #0 { +entry: + %r = tail call i32 @ext(i32 31) + ret i32 %r +} + +; CHECK-LABEL: : +; CHECK-NEXT: addi a0, zero, 31 +; CHECK-NEXT: auipc t1, 0 +; CHECK-NEXT: R_RISCV_CALL_PLT ext +; CHECK-NEXT: jalr zero, 0(t1) +define dso_local i32 @g() #1 { +entry: + %r = tail call i32 @ext(i32 31) + ret i32 %r +} + +attributes #0 = { nounwind "target-features"="+c,+relax" } +attributes #1 = { nounwind "target-features"="-c,-relax" } diff --git a/llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll b/llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll index e73249739f35..31ca8eab3350 100644 --- a/llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll +++ b/llvm/test/CodeGen/RISCV/riscv-codegenprepare-asm.ll @@ -10,8 +10,7 @@ define void @test1(ptr nocapture noundef %a, i32 noundef signext %n) { ; CHECK: # %bb.0: # %entry ; CHECK-NEXT: blez a1, .LBB0_3 ; CHECK-NEXT: # %bb.1: # %for.body.preheader -; CHECK-NEXT: slli a1, a1, 32 -; CHECK-NEXT: srli a1, a1, 30 +; CHECK-NEXT: slli a1, a1, 2 ; CHECK-NEXT: add a1, a0, a1 ; CHECK-NEXT: .LBB0_2: # %for.body ; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll index a56a81f5f793..769bfe8cd5ba 100644 --- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll +++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-int-shuffles.ll @@ -244,14 +244,14 @@ define <8 x i64> @vrgather_shuffle_vx_v8i64(<8 x i64> %x) { ; RV32-NEXT: addi a0, a0, %lo(.LCPI13_0) ; RV32-NEXT: vsetivli zero, 8, e64, m4, ta, mu ; RV32-NEXT: vle16.v v16, (a0) -; RV32-NEXT: vrgatherei16.vv v12, v8, v16 +; RV32-NEXT: vmv.v.i v20, 5 ; RV32-NEXT: lui a0, %hi(.LCPI13_1) ; RV32-NEXT: addi a0, a0, %lo(.LCPI13_1) -; RV32-NEXT: vle16.v v8, (a0) -; RV32-NEXT: li a0, 140 +; RV32-NEXT: vle16.v v17, (a0) +; RV32-NEXT: li a0, 115 ; RV32-NEXT: vmv.s.x v0, a0 -; RV32-NEXT: vmv.v.i v16, 5 -; RV32-NEXT: vrgatherei16.vv v12, v16, v8, v0.t +; RV32-NEXT: vrgatherei16.vv v12, v20, v16 +; RV32-NEXT: vrgatherei16.vv v12, v8, v17, v0.t ; RV32-NEXT: vmv.v.v v8, v12 ; RV32-NEXT: ret ; diff --git a/llvm/test/CodeGen/RISCV/sextw-removal.ll b/llvm/test/CodeGen/RISCV/sextw-removal.ll index 3babef93499c..e730092674e0 100644 --- a/llvm/test/CodeGen/RISCV/sextw-removal.ll +++ b/llvm/test/CodeGen/RISCV/sextw-removal.ll @@ -1,9 +1,9 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+f,+zknh -target-abi=lp64f \ +; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+f,+zknh,+v -target-abi=lp64f \ ; RUN: | FileCheck %s --check-prefixes=CHECK,RV64I -; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+zba,+zbb,+f,+zknh -target-abi=lp64f \ +; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+zba,+zbb,+f,+zknh,+v -target-abi=lp64f \ ; RUN: | FileCheck %s --check-prefixes=CHECK,RV64ZBB -; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+zba,+zbb,+f,+zknh -target-abi=lp64f \ +; RUN: llc < %s -mtriple=riscv64 -mattr=+m,+zba,+zbb,+f,+zknh,+v -target-abi=lp64f \ ; RUN: -riscv-disable-sextw-removal | FileCheck %s --check-prefix=NOREMOVAL define void @test1(i32 signext %arg, i32 signext %arg1) nounwind { @@ -1436,3 +1436,64 @@ bb7: ; preds = %bb2 } declare void @side_effect(i64) + +define void @test20( %arg, i32 signext %arg1) nounwind { +; CHECK-LABEL: test20: +; CHECK: # %bb.0: # %bb +; CHECK-NEXT: addi sp, sp, -32 +; CHECK-NEXT: sd ra, 24(sp) # 8-byte Folded Spill +; CHECK-NEXT: sd s0, 16(sp) # 8-byte Folded Spill +; CHECK-NEXT: sd s1, 8(sp) # 8-byte Folded Spill +; CHECK-NEXT: mv s0, a0 +; CHECK-NEXT: vsetivli zero, 1, e32, mf2, ta, ma +; CHECK-NEXT: vmv.x.s s1, v8 +; CHECK-NEXT: .LBB24_1: # %bb2 +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: mv a0, s1 +; CHECK-NEXT: call bar@plt +; CHECK-NEXT: sllw s1, s1, s0 +; CHECK-NEXT: bnez a0, .LBB24_1 +; CHECK-NEXT: # %bb.2: # %bb7 +; CHECK-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; CHECK-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; CHECK-NEXT: ld s1, 8(sp) # 8-byte Folded Reload +; CHECK-NEXT: addi sp, sp, 32 +; CHECK-NEXT: ret +; +; NOREMOVAL-LABEL: test20: +; NOREMOVAL: # %bb.0: # %bb +; NOREMOVAL-NEXT: addi sp, sp, -32 +; NOREMOVAL-NEXT: sd ra, 24(sp) # 8-byte Folded Spill +; NOREMOVAL-NEXT: sd s0, 16(sp) # 8-byte Folded Spill +; NOREMOVAL-NEXT: sd s1, 8(sp) # 8-byte Folded Spill +; NOREMOVAL-NEXT: mv s0, a0 +; NOREMOVAL-NEXT: vsetivli zero, 1, e32, mf2, ta, ma +; NOREMOVAL-NEXT: vmv.x.s s1, v8 +; NOREMOVAL-NEXT: .LBB24_1: # %bb2 +; NOREMOVAL-NEXT: # =>This Inner Loop Header: Depth=1 +; NOREMOVAL-NEXT: sext.w a0, s1 +; NOREMOVAL-NEXT: call bar@plt +; NOREMOVAL-NEXT: sllw s1, s1, s0 +; NOREMOVAL-NEXT: bnez a0, .LBB24_1 +; NOREMOVAL-NEXT: # %bb.2: # %bb7 +; NOREMOVAL-NEXT: ld ra, 24(sp) # 8-byte Folded Reload +; NOREMOVAL-NEXT: ld s0, 16(sp) # 8-byte Folded Reload +; NOREMOVAL-NEXT: ld s1, 8(sp) # 8-byte Folded Reload +; NOREMOVAL-NEXT: addi sp, sp, 32 +; NOREMOVAL-NEXT: ret +bb: + %i = call i32 @llvm.riscv.vmv.x.s.nxv1i32( %arg) + br label %bb2 + +bb2: ; preds = %bb2, %bb + %i3 = phi i32 [ %i, %bb ], [ %i5, %bb2 ] + %i4 = tail call signext i32 @bar(i32 signext %i3) + %i5 = shl i32 %i3, %arg1 + %i6 = icmp eq i32 %i4, 0 + br i1 %i6, label %bb7, label %bb2 + +bb7: ; preds = %bb2 + ret void +} + +declare i32 @llvm.riscv.vmv.x.s.nxv1i32( ) diff --git a/llvm/test/CodeGen/SPIRV/opencl/basic/get_global_offset.ll b/llvm/test/CodeGen/SPIRV/opencl/basic/get_global_offset.ll index da6015d23d05..127804671cee 100644 --- a/llvm/test/CodeGen/SPIRV/opencl/basic/get_global_offset.ll +++ b/llvm/test/CodeGen/SPIRV/opencl/basic/get_global_offset.ll @@ -12,6 +12,8 @@ ; CHECK: %[[#int64_ty:]] = OpTypeInt 64 0 ; CHECK: %[[#vec_ty:]] = OpTypeVector %[[#int64_ty]] 3 ; CHECK: %[[#func2_ty:]] = OpTypeFunction %[[#vec_ty]] +; CHECK: %[[#int32_ty:]] = OpTypeInt 32 0 +; CHECK: %[[#i32ptr_ty:]] = OpTypePointer CrossWorkgroup %[[#int32_ty]] ;; TODO: add 64-bit constant defs ; CHECK: %[[#f2_decl]] = OpFunction %[[#vec_ty]] Pure %[[#func2_ty]] ; CHECK: OpFunctionEnd @@ -31,21 +33,21 @@ entry: %0 = call spir_func <3 x i64> @BuiltInGlobalOffset() #1 %call = extractelement <3 x i64> %0, i32 0 %conv = trunc i64 %call to i32 -; CHECK: %[[#i1:]] = OpInBoundsPtrAccessChain %[[#iptr_ty]] %[[#outOffsets]] +; CHECK: %[[#i1:]] = OpInBoundsPtrAccessChain %[[#i32ptr_ty]] %[[#outOffsets]] ; CHECK: OpStore %[[#i1:]] %[[#]] Aligned 4 %arrayidx = getelementptr inbounds i32, i32 addrspace(1)* %outOffsets, i64 0 store i32 %conv, i32 addrspace(1)* %arrayidx, align 4 %1 = call spir_func <3 x i64> @BuiltInGlobalOffset() #1 %call1 = extractelement <3 x i64> %1, i32 1 %conv2 = trunc i64 %call1 to i32 -; CHECK: %[[#i2:]] = OpInBoundsPtrAccessChain %[[#iptr_ty]] %[[#outOffsets]] +; CHECK: %[[#i2:]] = OpInBoundsPtrAccessChain %[[#i32ptr_ty]] %[[#outOffsets]] ; CHECK: OpStore %[[#i2:]] %[[#]] Aligned 4 %arrayidx3 = getelementptr inbounds i32, i32 addrspace(1)* %outOffsets, i64 1 store i32 %conv2, i32 addrspace(1)* %arrayidx3, align 4 %2 = call spir_func <3 x i64> @BuiltInGlobalOffset() #1 %call4 = extractelement <3 x i64> %2, i32 2 %conv5 = trunc i64 %call4 to i32 -; CHECK: %[[#i3:]] = OpInBoundsPtrAccessChain %[[#iptr_ty]] %[[#outOffsets]] +; CHECK: %[[#i3:]] = OpInBoundsPtrAccessChain %[[#i32ptr_ty]] %[[#outOffsets]] ; CHECK: OpStore %[[#i3:]] %[[#]] Aligned 4 %arrayidx6 = getelementptr inbounds i32, i32 addrspace(1)* %outOffsets, i64 2 store i32 %conv5, i32 addrspace(1)* %arrayidx6, align 4 diff --git a/llvm/test/CodeGen/SPIRV/struct.ll b/llvm/test/CodeGen/SPIRV/struct.ll index 3c4fc81b9200..c3d2694cf929 100644 --- a/llvm/test/CodeGen/SPIRV/struct.ll +++ b/llvm/test/CodeGen/SPIRV/struct.ll @@ -3,11 +3,10 @@ %struct.ST = type { i32, i32, i32 } ; CHECK-DAG: OpName %[[#struct:]] "struct.ST" -; CHECK-DAG: %[[#char:]] = OpTypeInt 8 0 ; CHECK-DAG: %[[#int:]] = OpTypeInt 32 0 ; CHECK-DAG: %[[#struct]] = OpTypeStruct %[[#int]] %[[#int]] %[[#int]] ; CHECK-DAG: %[[#structP:]] = OpTypePointer Function %[[#struct]] -; CHECK-DAG: %[[#intP:]] = OpTypePointer Function %[[#char]] +; CHECK-DAG: %[[#intP:]] = OpTypePointer Function %[[#int]] ; CHECK-DAG: %[[#zero:]] = OpConstant %[[#int]] 0 ; CHECK-DAG: %[[#one:]] = OpConstant %[[#int]] 1 ; CHECK-DAG: %[[#two:]] = OpConstant %[[#int]] 2 diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-add-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-add-04.ll index 50d3eec15dbe..4af3793191b6 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-add-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-add-04.ll @@ -16,13 +16,12 @@ define i64 @f1(i64 %dummy, ptr %src, i64 %b) { ret i64 %res } -; Check addition of 1, which can use AGHI. +; Check addition of 1. define i64 @f2(i64 %dummy, ptr %src) { ; CHECK-LABEL: f2: ; CHECK: lg %r2, 0(%r3) ; CHECK: [[LABEL:\.[^:]*]]: -; CHECK: lgr %r0, %r2 -; CHECK: aghi %r0, 1 +; CHECK: la %r0, 1(%r2) ; CHECK: csg %r2, %r0, 0(%r3) ; CHECK: jl [[LABEL]] ; CHECK: br %r14 @@ -30,82 +29,64 @@ define i64 @f2(i64 %dummy, ptr %src) { ret i64 %res } -; Check the high end of the AGHI range. +; Check use of LAY. define i64 @f3(i64 %dummy, ptr %src) { ; CHECK-LABEL: f3: -; CHECK: aghi %r0, 32767 +; CHECK: lay %r0, 32767(%r2) ; CHECK: br %r14 %res = atomicrmw add ptr %src, i64 32767 seq_cst ret i64 %res } -; Check the next value up, which must use AGFI. +; Check the high end of the AGFI range. define i64 @f4(i64 %dummy, ptr %src) { ; CHECK-LABEL: f4: -; CHECK: agfi %r0, 32768 +; CHECK: agfi %r0, 2147483647 ; CHECK: br %r14 - %res = atomicrmw add ptr %src, i64 32768 seq_cst + %res = atomicrmw add ptr %src, i64 2147483647 seq_cst ret i64 %res } -; Check the high end of the AGFI range. +; Check the next value up, which uses an ALGFI. define i64 @f5(i64 %dummy, ptr %src) { ; CHECK-LABEL: f5: -; CHECK: agfi %r0, 2147483647 +; CHECK: algfi %r0, 2147483648 ; CHECK: br %r14 - %res = atomicrmw add ptr %src, i64 2147483647 seq_cst + %res = atomicrmw add ptr %src, i64 2147483648 seq_cst ret i64 %res } -; Check the next value up, which must use a register addition. +; Check addition of -1, which can use LAY. define i64 @f6(i64 %dummy, ptr %src) { ; CHECK-LABEL: f6: -; CHECK: agr +; CHECK: lay %r0, -1(%r2) ; CHECK: br %r14 - %res = atomicrmw add ptr %src, i64 2147483648 seq_cst + %res = atomicrmw add ptr %src, i64 -1 seq_cst ret i64 %res } -; Check addition of -1, which can use AGHI. +; LAY still OK. define i64 @f7(i64 %dummy, ptr %src) { ; CHECK-LABEL: f7: -; CHECK: aghi %r0, -1 -; CHECK: br %r14 - %res = atomicrmw add ptr %src, i64 -1 seq_cst - ret i64 %res -} - -; Check the low end of the AGHI range. -define i64 @f8(i64 %dummy, ptr %src) { -; CHECK-LABEL: f8: -; CHECK: aghi %r0, -32768 -; CHECK: br %r14 - %res = atomicrmw add ptr %src, i64 -32768 seq_cst - ret i64 %res -} - -; Check the next value down, which must use AGFI instead. -define i64 @f9(i64 %dummy, ptr %src) { -; CHECK-LABEL: f9: -; CHECK: agfi %r0, -32769 +; CHECK: lay %r0, -32769(%r2) ; CHECK: br %r14 %res = atomicrmw add ptr %src, i64 -32769 seq_cst ret i64 %res } ; Check the low end of the AGFI range. -define i64 @f10(i64 %dummy, ptr %src) { -; CHECK-LABEL: f10: +define i64 @f8(i64 %dummy, ptr %src) { +; CHECK-LABEL: f8: ; CHECK: agfi %r0, -2147483648 ; CHECK: br %r14 %res = atomicrmw add ptr %src, i64 -2147483648 seq_cst ret i64 %res } -; Check the next value down, which must use a register addition. -define i64 @f11(i64 %dummy, ptr %src) { -; CHECK-LABEL: f11: -; CHECK: agr +; Check the next value down, which uses an SLGFI. +define i64 @f9(i64 %dummy, ptr %src) { +; CHECK-LABEL: f9: +; CHECK: slgfi %r0, 2147483649 ; CHECK: br %r14 %res = atomicrmw add ptr %src, i64 -2147483649 seq_cst ret i64 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-and-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-and-03.ll index 03ed2404dcc1..96c82e6b1eae 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-and-03.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-and-03.ll @@ -33,7 +33,7 @@ define i32 @f2(i32 %dummy, ptr %src) { ; Check ANDs of the low end of the NILH range. define i32 @f3(i32 %dummy, ptr %src) { ; CHECK-LABEL: f3: -; CHECK: nilh %r0, 0 +; CHECK: llhr %r0, %r2 ; CHECK: br %r14 %res = atomicrmw and ptr %src, i32 65535 seq_cst ret i32 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-and-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-and-04.ll index 00b6fd93ad5b..9647548c842b 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-and-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-and-04.ll @@ -16,11 +16,10 @@ define i64 @f1(i64 %dummy, ptr %src, i64 %b) { ret i64 %res } -; Check ANDs of 1, which are done using a register. (We could use RISBG -; instead, but that isn't implemented yet.) +; Check ANDs of 1, which are done using a register. define i64 @f2(i64 %dummy, ptr %src) { ; CHECK-LABEL: f2: -; CHECK: ngr +; CHECK: risbg ; CHECK: br %r14 %res = atomicrmw and ptr %src, i64 1 seq_cst ret i64 %res @@ -56,7 +55,7 @@ define i64 @f4(i64 %dummy, ptr %src) { ; Check the next value up, which must use a register. define i64 @f5(i64 %dummy, ptr %src) { ; CHECK-LABEL: f5: -; CHECK: ngr +; CHECK: risbg ; CHECK: br %r14 %res = atomicrmw and ptr %src, i64 12884901888 seq_cst ret i64 %res @@ -74,7 +73,7 @@ define i64 @f6(i64 %dummy, ptr %src) { ; Check the next value up, which must use a register. define i64 @f7(i64 %dummy, ptr %src) { ; CHECK-LABEL: f7: -; CHECK: ngr +; CHECK: risbg ; CHECK: br %r14 %res = atomicrmw and ptr %src, i64 281474976710656 seq_cst ret i64 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll index d633c2d74e3a..d107e5d1dc2e 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-03.ll @@ -1,21 +1,31 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; Test 32-bit atomic minimum and maximum. Here we match the z10 versions, ; which can't use LOCR. ; ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s +; Todo: If-converter no longer producing CondReturns (with AtomicExpand pass). + ; Check signed minimum. define i32 @f1(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f1: -; CHECK: l %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: cs %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP]]: -; CHECK: lr [[NEW]], %r2 -; CHECK: crjle %r2, %r4, [[KEEP:\..*]] -; CHECK: lr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB0_2 +; CHECK-NEXT: .LBB0_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB0_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB0_4 +; CHECK-NEXT: .LBB0_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB0_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB0_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB0_1 +; CHECK-NEXT: .LBB0_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw min ptr %src, i32 %b seq_cst ret i32 %res } @@ -23,16 +33,23 @@ define i32 @f1(i32 %dummy, ptr %src, i32 %b) { ; Check signed maximum. define i32 @f2(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f2: -; CHECK: l %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: cs %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP]]: -; CHECK: lr [[NEW]], %r2 -; CHECK: crjhe %r2, %r4, [[KEEP:\..*]] -; CHECK: lr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB1_2 +; CHECK-NEXT: .LBB1_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB1_4 +; CHECK-NEXT: .LBB1_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjh %r2, %r4, .LBB1_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB1_1 +; CHECK-NEXT: .LBB1_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw max ptr %src, i32 %b seq_cst ret i32 %res } @@ -40,16 +57,23 @@ define i32 @f2(i32 %dummy, ptr %src, i32 %b) { ; Check unsigned minimum. define i32 @f3(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f3: -; CHECK: l %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: cs %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP]]: -; CHECK: lr [[NEW]], %r2 -; CHECK: clrjle %r2, %r4, [[KEEP:\..*]] -; CHECK: lr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB2_2 +; CHECK-NEXT: .LBB2_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB2_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB2_4 +; CHECK-NEXT: .LBB2_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: clrjle %r2, %r4, .LBB2_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB2_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB2_1 +; CHECK-NEXT: .LBB2_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw umin ptr %src, i32 %b seq_cst ret i32 %res } @@ -57,16 +81,23 @@ define i32 @f3(i32 %dummy, ptr %src, i32 %b) { ; Check unsigned maximum. define i32 @f4(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f4: -; CHECK: l %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: cs %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP]]: -; CHECK: lr [[NEW]], %r2 -; CHECK: clrjhe %r2, %r4, [[KEEP:\..*]] -; CHECK: lr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB3_2 +; CHECK-NEXT: .LBB3_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB3_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB3_4 +; CHECK-NEXT: .LBB3_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: clrjh %r2, %r4, .LBB3_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB3_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB3_1 +; CHECK-NEXT: .LBB3_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw umax ptr %src, i32 %b seq_cst ret i32 %res } @@ -74,9 +105,23 @@ define i32 @f4(i32 %dummy, ptr %src, i32 %b) { ; Check the high end of the aligned CS range. define i32 @f5(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f5: -; CHECK: l %r2, 4092(%r3) -; CHECK: cs %r2, {{%r[0-9]+}}, 4092(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 4092(%r3) +; CHECK-NEXT: j .LBB4_2 +; CHECK-NEXT: .LBB4_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB4_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 4092(%r3) +; CHECK-NEXT: je .LBB4_4 +; CHECK-NEXT: .LBB4_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB4_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB4_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB4_1 +; CHECK-NEXT: .LBB4_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 1023 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -85,9 +130,23 @@ define i32 @f5(i32 %dummy, ptr %src, i32 %b) { ; Check the next word up, which requires CSY. define i32 @f6(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f6: -; CHECK: ly %r2, 4096(%r3) -; CHECK: csy %r2, {{%r[0-9]+}}, 4096(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: ly %r2, 4096(%r3) +; CHECK-NEXT: j .LBB5_2 +; CHECK-NEXT: .LBB5_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB5_2 Depth=1 +; CHECK-NEXT: csy %r2, %r0, 4096(%r3) +; CHECK-NEXT: je .LBB5_4 +; CHECK-NEXT: .LBB5_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB5_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB5_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB5_1 +; CHECK-NEXT: .LBB5_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 1024 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -96,9 +155,23 @@ define i32 @f6(i32 %dummy, ptr %src, i32 %b) { ; Check the high end of the aligned CSY range. define i32 @f7(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f7: -; CHECK: ly %r2, 524284(%r3) -; CHECK: csy %r2, {{%r[0-9]+}}, 524284(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: ly %r2, 524284(%r3) +; CHECK-NEXT: j .LBB6_2 +; CHECK-NEXT: .LBB6_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB6_2 Depth=1 +; CHECK-NEXT: csy %r2, %r0, 524284(%r3) +; CHECK-NEXT: je .LBB6_4 +; CHECK-NEXT: .LBB6_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB6_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB6_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB6_1 +; CHECK-NEXT: .LBB6_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 131071 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -107,10 +180,24 @@ define i32 @f7(i32 %dummy, ptr %src, i32 %b) { ; Check the next word up, which needs separate address logic. define i32 @f8(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f8: -; CHECK: agfi %r3, 524288 -; CHECK: l %r2, 0(%r3) -; CHECK: cs %r2, {{%r[0-9]+}}, 0(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: agfi %r3, 524288 +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB7_2 +; CHECK-NEXT: .LBB7_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB7_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB7_4 +; CHECK-NEXT: .LBB7_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB7_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB7_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB7_1 +; CHECK-NEXT: .LBB7_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 131072 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -119,9 +206,23 @@ define i32 @f8(i32 %dummy, ptr %src, i32 %b) { ; Check the high end of the negative aligned CSY range. define i32 @f9(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f9: -; CHECK: ly %r2, -4(%r3) -; CHECK: csy %r2, {{%r[0-9]+}}, -4(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: ly %r2, -4(%r3) +; CHECK-NEXT: j .LBB8_2 +; CHECK-NEXT: .LBB8_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB8_2 Depth=1 +; CHECK-NEXT: csy %r2, %r0, -4(%r3) +; CHECK-NEXT: je .LBB8_4 +; CHECK-NEXT: .LBB8_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB8_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB8_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB8_1 +; CHECK-NEXT: .LBB8_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 -1 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -130,9 +231,23 @@ define i32 @f9(i32 %dummy, ptr %src, i32 %b) { ; Check the low end of the CSY range. define i32 @f10(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f10: -; CHECK: ly %r2, -524288(%r3) -; CHECK: csy %r2, {{%r[0-9]+}}, -524288(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: ly %r2, -524288(%r3) +; CHECK-NEXT: j .LBB9_2 +; CHECK-NEXT: .LBB9_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB9_2 Depth=1 +; CHECK-NEXT: csy %r2, %r0, -524288(%r3) +; CHECK-NEXT: je .LBB9_4 +; CHECK-NEXT: .LBB9_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB9_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB9_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB9_1 +; CHECK-NEXT: .LBB9_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 -131072 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -141,10 +256,24 @@ define i32 @f10(i32 %dummy, ptr %src, i32 %b) { ; Check the next word down, which needs separate address logic. define i32 @f11(i32 %dummy, ptr %src, i32 %b) { ; CHECK-LABEL: f11: -; CHECK: agfi %r3, -524292 -; CHECK: l %r2, 0(%r3) -; CHECK: cs %r2, {{%r[0-9]+}}, 0(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: agfi %r3, -524292 +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB10_2 +; CHECK-NEXT: .LBB10_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB10_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB10_4 +; CHECK-NEXT: .LBB10_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r4, .LBB10_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB10_2 Depth=1 +; CHECK-NEXT: lr %r0, %r4 +; CHECK-NEXT: j .LBB10_1 +; CHECK-NEXT: .LBB10_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i32, ptr %src, i64 -131073 %res = atomicrmw min ptr %ptr, i32 %b seq_cst ret i32 %res @@ -153,10 +282,24 @@ define i32 @f11(i32 %dummy, ptr %src, i32 %b) { ; Check that indexed addresses are not allowed. define i32 @f12(i32 %dummy, i64 %base, i64 %index, i32 %b) { ; CHECK-LABEL: f12: -; CHECK: agr %r3, %r4 -; CHECK: l %r2, 0(%r3) -; CHECK: cs %r2, {{%r[0-9]+}}, 0(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 0(%r4,%r3) +; CHECK-NEXT: agr %r3, %r4 +; CHECK-NEXT: j .LBB11_2 +; CHECK-NEXT: .LBB11_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB11_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB11_4 +; CHECK-NEXT: .LBB11_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: crjle %r2, %r5, .LBB11_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB11_2 Depth=1 +; CHECK-NEXT: lr %r0, %r5 +; CHECK-NEXT: j .LBB11_1 +; CHECK-NEXT: .LBB11_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %add = add i64 %base, %index %ptr = inttoptr i64 %add to ptr %res = atomicrmw min ptr %ptr, i32 %b seq_cst @@ -166,16 +309,23 @@ define i32 @f12(i32 %dummy, i64 %base, i64 %index, i32 %b) { ; Check that constants are handled. define i32 @f13(i32 %dummy, ptr %ptr) { ; CHECK-LABEL: f13: -; CHECK: lhi [[LIMIT:%r[0-9]+]], 42 -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: cs %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP]]: -; CHECK: lr [[NEW]], %r2 -; CHECK: crjle %r2, [[LIMIT]], [[KEEP:\..*]] -; CHECK: lhi [[NEW]], 42 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: l %r2, 0(%r3) +; CHECK-NEXT: j .LBB12_2 +; CHECK-NEXT: .LBB12_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB12_2 Depth=1 +; CHECK-NEXT: cs %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB12_4 +; CHECK-NEXT: .LBB12_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lr %r0, %r2 +; CHECK-NEXT: cijl %r2, 43, .LBB12_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB12_2 Depth=1 +; CHECK-NEXT: lhi %r0, 42 +; CHECK-NEXT: j .LBB12_1 +; CHECK-NEXT: .LBB12_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw min ptr %ptr, i32 42 seq_cst ret i32 %res } diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll index 64e76e0a90ea..9352118a32f8 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-minmax-04.ll @@ -1,21 +1,31 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; Test 64-bit atomic minimum and maximum. Here we match the z10 versions, ; which can't use LOCGR. ; ; RUN: llc < %s -mtriple=s390x-linux-gnu -mcpu=z10 | FileCheck %s +; Todo: If-converter no longer producing CondReturns (with AtomicExpand pass). + ; Check signed minimum. define i64 @f1(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f1: -; CHECK: lg %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: csg %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP:\.[^:]*]]: -; CHECK: lgr [[NEW:%r[0-9]+]], %r2 -; CHECK: cgrjle %r2, %r4, [[KEEP:\..*]] -; CHECK: lgr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB0_2 +; CHECK-NEXT: .LBB0_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB0_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB0_4 +; CHECK-NEXT: .LBB0_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjle %r2, %r4, .LBB0_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB0_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB0_1 +; CHECK-NEXT: .LBB0_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw min ptr %src, i64 %b seq_cst ret i64 %res } @@ -23,16 +33,23 @@ define i64 @f1(i64 %dummy, ptr %src, i64 %b) { ; Check signed maximum. define i64 @f2(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f2: -; CHECK: lg %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: csg %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP:\.[^:]*]]: -; CHECK: lgr [[NEW:%r[0-9]+]], %r2 -; CHECK: cgrjhe %r2, %r4, [[KEEP:\..*]] -; CHECK: lgr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB1_2 +; CHECK-NEXT: .LBB1_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB1_4 +; CHECK-NEXT: .LBB1_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjh %r2, %r4, .LBB1_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB1_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB1_1 +; CHECK-NEXT: .LBB1_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw max ptr %src, i64 %b seq_cst ret i64 %res } @@ -40,16 +57,23 @@ define i64 @f2(i64 %dummy, ptr %src, i64 %b) { ; Check unsigned minimum. define i64 @f3(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f3: -; CHECK: lg %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: csg %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP:\.[^:]*]]: -; CHECK: lgr [[NEW:%r[0-9]+]], %r2 -; CHECK: clgrjle %r2, %r4, [[KEEP:\..*]] -; CHECK: lgr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB2_2 +; CHECK-NEXT: .LBB2_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB2_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB2_4 +; CHECK-NEXT: .LBB2_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: clgrjle %r2, %r4, .LBB2_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB2_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB2_1 +; CHECK-NEXT: .LBB2_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw umin ptr %src, i64 %b seq_cst ret i64 %res } @@ -57,16 +81,23 @@ define i64 @f3(i64 %dummy, ptr %src, i64 %b) { ; Check unsigned maximum. define i64 @f4(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f4: -; CHECK: lg %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: csg %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP:\.[^:]*]]: -; CHECK: lgr [[NEW:%r[0-9]+]], %r2 -; CHECK: clgrjhe %r2, %r4, [[KEEP:\..*]] -; CHECK: lgr [[NEW]], %r4 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB3_2 +; CHECK-NEXT: .LBB3_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB3_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB3_4 +; CHECK-NEXT: .LBB3_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: clgrjh %r2, %r4, .LBB3_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB3_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB3_1 +; CHECK-NEXT: .LBB3_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw umax ptr %src, i64 %b seq_cst ret i64 %res } @@ -74,9 +105,23 @@ define i64 @f4(i64 %dummy, ptr %src, i64 %b) { ; Check the high end of the aligned CSG range. define i64 @f5(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f5: -; CHECK: lg %r2, 524280(%r3) -; CHECK: csg %r2, {{%r[0-9]+}}, 524280(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 524280(%r3) +; CHECK-NEXT: j .LBB4_2 +; CHECK-NEXT: .LBB4_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB4_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 524280(%r3) +; CHECK-NEXT: je .LBB4_4 +; CHECK-NEXT: .LBB4_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjle %r2, %r4, .LBB4_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB4_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB4_1 +; CHECK-NEXT: .LBB4_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i64, ptr %src, i64 65535 %res = atomicrmw min ptr %ptr, i64 %b seq_cst ret i64 %res @@ -85,10 +130,24 @@ define i64 @f5(i64 %dummy, ptr %src, i64 %b) { ; Check the next doubleword up, which requires separate address logic. define i64 @f6(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f6: -; CHECK: agfi %r3, 524288 -; CHECK: lg %r2, 0(%r3) -; CHECK: csg %r2, {{%r[0-9]+}}, 0(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: agfi %r3, 524288 +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB5_2 +; CHECK-NEXT: .LBB5_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB5_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB5_4 +; CHECK-NEXT: .LBB5_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjle %r2, %r4, .LBB5_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB5_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB5_1 +; CHECK-NEXT: .LBB5_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i64, ptr %src, i64 65536 %res = atomicrmw min ptr %ptr, i64 %b seq_cst ret i64 %res @@ -97,9 +156,23 @@ define i64 @f6(i64 %dummy, ptr %src, i64 %b) { ; Check the low end of the CSG range. define i64 @f7(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f7: -; CHECK: lg %r2, -524288(%r3) -; CHECK: csg %r2, {{%r[0-9]+}}, -524288(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, -524288(%r3) +; CHECK-NEXT: j .LBB6_2 +; CHECK-NEXT: .LBB6_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB6_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, -524288(%r3) +; CHECK-NEXT: je .LBB6_4 +; CHECK-NEXT: .LBB6_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjle %r2, %r4, .LBB6_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB6_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB6_1 +; CHECK-NEXT: .LBB6_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i64, ptr %src, i64 -65536 %res = atomicrmw min ptr %ptr, i64 %b seq_cst ret i64 %res @@ -108,10 +181,24 @@ define i64 @f7(i64 %dummy, ptr %src, i64 %b) { ; Check the next doubleword down, which requires separate address logic. define i64 @f8(i64 %dummy, ptr %src, i64 %b) { ; CHECK-LABEL: f8: -; CHECK: agfi %r3, -524296 -; CHECK: lg %r2, 0(%r3) -; CHECK: csg %r2, {{%r[0-9]+}}, 0(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: agfi %r3, -524296 +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB7_2 +; CHECK-NEXT: .LBB7_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB7_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB7_4 +; CHECK-NEXT: .LBB7_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjle %r2, %r4, .LBB7_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB7_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r4 +; CHECK-NEXT: j .LBB7_1 +; CHECK-NEXT: .LBB7_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %ptr = getelementptr i64, ptr %src, i64 -65537 %res = atomicrmw min ptr %ptr, i64 %b seq_cst ret i64 %res @@ -120,10 +207,24 @@ define i64 @f8(i64 %dummy, ptr %src, i64 %b) { ; Check that indexed addresses are not allowed. define i64 @f9(i64 %dummy, i64 %base, i64 %index, i64 %b) { ; CHECK-LABEL: f9: -; CHECK: agr %r3, %r4 -; CHECK: lg %r2, 0(%r3) -; CHECK: csg %r2, {{%r[0-9]+}}, 0(%r3) -; CHECK: ber %r14 +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 0(%r4,%r3) +; CHECK-NEXT: agr %r3, %r4 +; CHECK-NEXT: j .LBB8_2 +; CHECK-NEXT: .LBB8_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB8_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB8_4 +; CHECK-NEXT: .LBB8_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgrjle %r2, %r5, .LBB8_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB8_2 Depth=1 +; CHECK-NEXT: lgr %r0, %r5 +; CHECK-NEXT: j .LBB8_1 +; CHECK-NEXT: .LBB8_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %add = add i64 %base, %index %ptr = inttoptr i64 %add to ptr %res = atomicrmw min ptr %ptr, i64 %b seq_cst @@ -133,17 +234,23 @@ define i64 @f9(i64 %dummy, i64 %base, i64 %index, i64 %b) { ; Check that constants are handled. define i64 @f10(i64 %dummy, ptr %ptr) { ; CHECK-LABEL: f10: -; CHECK-DAG: lghi [[LIMIT:%r[0-9]+]], 42 -; CHECK-DAG: lg %r2, 0(%r3) -; CHECK: j [[LOOP:\.[^:]*]] -; CHECK: [[BB1:\.[^:]*]]: -; CHECK: csg %r2, [[NEW:%r[0-9]+]], 0(%r3) -; CHECK: ber %r14 -; CHECK: [[LOOP:\.[^:]*]]: -; CHECK: lgr [[NEW:%r[0-9]+]], %r2 -; CHECK: cgrjle %r2, [[LIMIT]], [[KEEP:\..*]] -; CHECK: lghi [[NEW]], 42 -; CHECK: j [[BB1]] +; CHECK: # %bb.0: +; CHECK-NEXT: lg %r2, 0(%r3) +; CHECK-NEXT: j .LBB9_2 +; CHECK-NEXT: .LBB9_1: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB9_2 Depth=1 +; CHECK-NEXT: csg %r2, %r0, 0(%r3) +; CHECK-NEXT: je .LBB9_4 +; CHECK-NEXT: .LBB9_2: # %atomicrmw.start +; CHECK-NEXT: # =>This Inner Loop Header: Depth=1 +; CHECK-NEXT: lgr %r0, %r2 +; CHECK-NEXT: cgijl %r2, 43, .LBB9_1 +; CHECK-NEXT: # %bb.3: # %atomicrmw.start +; CHECK-NEXT: # in Loop: Header=BB9_2 Depth=1 +; CHECK-NEXT: lghi %r0, 42 +; CHECK-NEXT: j .LBB9_1 +; CHECK-NEXT: .LBB9_4: # %atomicrmw.end +; CHECK-NEXT: br %r14 %res = atomicrmw min ptr %ptr, i64 42 seq_cst ret i64 %res } diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-nand-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-nand-03.ll index 323eafb3e5b3..8e9870f13013 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-nand-03.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-nand-03.ll @@ -17,14 +17,14 @@ define i32 @f1(i32 %dummy, ptr %src, i32 %b) { ret i32 %res } -; Check NANDs of 1. +; Check NANDs with different constant operands. define i32 @f2(i32 %dummy, ptr %src) { ; CHECK-LABEL: f2: ; CHECK: l %r2, 0(%r3) ; CHECK: [[LABEL:\.[^ ]*]]: ; CHECK: lr %r0, %r2 -; CHECK: nilf %r0, 1 ; CHECK: xilf %r0, 4294967295 +; CHECK: oilf %r0, 4294967294 ; CHECK: cs %r2, %r0, 0(%r3) ; CHECK: jl [[LABEL]] ; CHECK: br %r14 @@ -32,61 +32,55 @@ define i32 @f2(i32 %dummy, ptr %src) { ret i32 %res } -; Check NANDs of the low end of the NILH range. define i32 @f3(i32 %dummy, ptr %src) { ; CHECK-LABEL: f3: -; CHECK: nilh %r0, 0 ; CHECK: xilf %r0, 4294967295 +; CHECK: oilh %r0, 65535 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i32 65535 seq_cst ret i32 %res } -; Check the next value up, which must use NILF. define i32 @f4(i32 %dummy, ptr %src) { ; CHECK-LABEL: f4: -; CHECK: nilf %r0, 65536 ; CHECK: xilf %r0, 4294967295 +; CHECK: oilf %r0, 4294901759 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i32 65536 seq_cst ret i32 %res } -; Check the largest useful NILL value. define i32 @f5(i32 %dummy, ptr %src) { ; CHECK-LABEL: f5: -; CHECK: nill %r0, 65534 ; CHECK: xilf %r0, 4294967295 +; CHECK: oill %r0, 1 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i32 -2 seq_cst ret i32 %res } -; Check the low end of the NILL range. define i32 @f6(i32 %dummy, ptr %src) { ; CHECK-LABEL: f6: -; CHECK: nill %r0, 0 ; CHECK: xilf %r0, 4294967295 +; CHECK: oill %r0, 65535 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i32 -65536 seq_cst ret i32 %res } -; Check the largest useful NILH value, which is one less than the above. define i32 @f7(i32 %dummy, ptr %src) { ; CHECK-LABEL: f7: -; CHECK: nilh %r0, 65534 ; CHECK: xilf %r0, 4294967295 +; CHECK: oilh %r0, 1 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i32 -65537 seq_cst ret i32 %res } -; Check the highest useful NILF value, which is one less than the above. define i32 @f8(i32 %dummy, ptr %src) { ; CHECK-LABEL: f8: -; CHECK: nilf %r0, 4294901758 ; CHECK: xilf %r0, 4294967295 +; CHECK: oilf %r0, 65537 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i32 -65538 seq_cst ret i32 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-nand-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-nand-04.ll index b37030255d7c..3ff259d5576c 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-nand-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-nand-04.ll @@ -18,40 +18,38 @@ define i64 @f1(i64 %dummy, ptr %src, i64 %b) { ret i64 %res } -; Check NANDs of 1, which are done using a register. (We could use RISBG -; instead, but that isn't implemented yet.) +; Check NANDs of 1, which are done using a register. define i64 @f2(i64 %dummy, ptr %src) { ; CHECK-LABEL: f2: -; CHECK: ngr +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihf %r0, 4294967295 +; CHECK: oilf %r0, 4294967294 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 1 seq_cst ret i64 %res } -; Check the equivalent of NIHF with 1, which can use RISBG instead. define i64 @f3(i64 %dummy, ptr %src) { ; CHECK-LABEL: f3: ; CHECK: lg %r2, 0(%r3) ; CHECK: [[LABEL:\.[^:]*]]: -; CHECK: risbg %r0, %r2, 31, 191, 0 -; CHECK: lcgr %r0, %r0 +; CHECK: lcgr %r0, %r2 ; CHECK: aghi %r0, -1 -; CHECK: csg %r2, %r0, 0(%r3) +; CHECK: oihf %r0, 4294967294 ; CHECK: jl [[LABEL]] ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 8589934591 seq_cst ret i64 %res } -; Check the lowest NIHF value outside the range of RISBG. define i64 @f4(i64 %dummy, ptr %src) { ; CHECK-LABEL: f4: ; CHECK: lg %r2, 0(%r3) ; CHECK: [[LABEL:\.[^:]*]]: -; CHECK: lgr %r0, %r2 -; CHECK: nihf %r0, 2 -; CHECK: lcgr %r0, %r0 +; CHECK: lcgr %r0, %r2 ; CHECK: aghi %r0, -1 +; CHECK: oihf %r0, 4294967293 ; CHECK: csg %r2, %r0, 0(%r3) ; CHECK: jl [[LABEL]] ; CHECK: br %r14 @@ -59,118 +57,133 @@ define i64 @f4(i64 %dummy, ptr %src) { ret i64 %res } -; Check the next value up, which must use a register. define i64 @f5(i64 %dummy, ptr %src) { ; CHECK-LABEL: f5: -; CHECK: ngr +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihf %r0, 4294967292 +; CHECK: oilf %r0, 4294967295 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 12884901888 seq_cst ret i64 %res } -; Check the lowest NIHH value outside the range of RISBG. define i64 @f6(i64 %dummy, ptr %src) { ; CHECK-LABEL: f6: -; CHECK: nihh {{%r[0-5]}}, 2 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihh %r0, 65533 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 844424930131967 seq_cst ret i64 %res } -; Check the next value up, which must use a register. define i64 @f7(i64 %dummy, ptr %src) { ; CHECK-LABEL: f7: -; CHECK: ngr +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihf %r0, 4294901759 +; CHECK: oilf %r0, 4294967295 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 281474976710656 seq_cst ret i64 %res } -; Check the highest NILL value outside the range of RISBG. define i64 @f8(i64 %dummy, ptr %src) { ; CHECK-LABEL: f8: -; CHECK: nill {{%r[0-5]}}, 65530 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oill %r0, 5 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -6 seq_cst ret i64 %res } -; Check the lowest NILL value outside the range of RISBG. define i64 @f9(i64 %dummy, ptr %src) { ; CHECK-LABEL: f9: -; CHECK: nill {{%r[0-5]}}, 2 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oill %r0, 65533 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -65534 seq_cst ret i64 %res } -; Check the highest useful NILF value. define i64 @f10(i64 %dummy, ptr %src) { ; CHECK-LABEL: f10: -; CHECK: nilf {{%r[0-5]}}, 4294901758 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oilf %r0, 65537 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -65538 seq_cst ret i64 %res } -; Check the highest NILH value outside the range of RISBG. define i64 @f11(i64 %dummy, ptr %src) { ; CHECK-LABEL: f11: -; CHECK: nilh {{%r[0-5]}}, 65530 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oilh %r0, 5 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -327681 seq_cst ret i64 %res } -; Check the lowest NILH value outside the range of RISBG. define i64 @f12(i64 %dummy, ptr %src) { ; CHECK-LABEL: f12: -; CHECK: nilh {{%r[0-5]}}, 2 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oilh %r0, 65533 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -4294770689 seq_cst ret i64 %res } -; Check the lowest NILF value outside the range of RISBG. define i64 @f13(i64 %dummy, ptr %src) { ; CHECK-LABEL: f13: -; CHECK: nilf {{%r[0-5]}}, 2 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oilf %r0, 4294967293 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -4294967294 seq_cst ret i64 %res } -; Check the highest NIHL value outside the range of RISBG. define i64 @f14(i64 %dummy, ptr %src) { ; CHECK-LABEL: f14: -; CHECK: nihl {{%r[0-5]}}, 65530 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihl %r0, 5 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -21474836481 seq_cst ret i64 %res } -; Check the lowest NIHL value outside the range of RISBG. define i64 @f15(i64 %dummy, ptr %src) { ; CHECK-LABEL: f15: -; CHECK: nihl {{%r[0-5]}}, 2 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihl %r0, 65533 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -281462091808769 seq_cst ret i64 %res } -; Check the highest NIHH value outside the range of RISBG. define i64 @f16(i64 %dummy, ptr %src) { ; CHECK-LABEL: f16: -; CHECK: nihh {{%r[0-5]}}, 65530 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihh %r0, 5 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -1407374883553281 seq_cst ret i64 %res } -; Check the highest useful NIHF value. define i64 @f17(i64 %dummy, ptr %src) { ; CHECK-LABEL: f17: -; CHECK: nihf {{%r[0-5]}}, 4294901758 +; CHECK: lcgr %r0, %r2 +; CHECK: aghi %r0, -1 +; CHECK: oihf %r0, 65537 ; CHECK: br %r14 %res = atomicrmw nand ptr %src, i64 -281479271677953 seq_cst ret i64 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-or-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-or-04.ll index a1322df49dbf..e29097b10e16 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-or-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-or-04.ll @@ -93,11 +93,11 @@ define i64 @f9(i64 %dummy, ptr %src) { ret i64 %res } -; Check the next value up, which must use a register. (We could use -; combinations of OIH* and OIL* instead, but that isn't implemented.) +; Check the next value up, which must use a register. define i64 @f10(i64 %dummy, ptr %src) { ; CHECK-LABEL: f10: -; CHECK: ogr +; CHECK: oihl %r0, 1 +; CHECK: oill %r0, 1 ; CHECK: br %r14 %res = atomicrmw or ptr %src, i64 4294967297 seq_cst ret i64 %res @@ -139,10 +139,11 @@ define i64 @f14(i64 %dummy, ptr %src) { ret i64 %res } -; Check the next value up, which must use a register. +; Check the next value up. define i64 @f15(i64 %dummy, ptr %src) { ; CHECK-LABEL: f15: -; CHECK: ogr +; CHECK: oihh %r0, 65535 +; CHECK: oill %r0, 1 ; CHECK: br %r14 %res = atomicrmw or ptr %src, i64 18446462598732840961 seq_cst ret i64 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-sub-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-sub-04.ll index 5d23d4e9ca15..d18c72f3b41e 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-sub-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-sub-04.ll @@ -16,13 +16,12 @@ define i64 @f1(i64 %dummy, ptr %src, i64 %b) { ret i64 %res } -; Check subtraction of 1, which can use AGHI. +; Check subtraction of 1. define i64 @f2(i64 %dummy, ptr %src) { ; CHECK-LABEL: f2: ; CHECK: lg %r2, 0(%r3) ; CHECK: [[LABEL:\.[^:]*]]: -; CHECK: lgr %r0, %r2 -; CHECK: aghi %r0, -1 +; CHECK: lay %r0, -1(%r2) ; CHECK: csg %r2, %r0, 0(%r3) ; CHECK: jl [[LABEL]] ; CHECK: br %r14 @@ -30,82 +29,64 @@ define i64 @f2(i64 %dummy, ptr %src) { ret i64 %res } -; Check the low end of the AGHI range. +; Check use of LAY. define i64 @f3(i64 %dummy, ptr %src) { ; CHECK-LABEL: f3: -; CHECK: aghi %r0, -32768 +; CHECK: lay %r0, -32768(%r2) ; CHECK: br %r14 %res = atomicrmw sub ptr %src, i64 32768 seq_cst ret i64 %res } -; Check the next value up, which must use AGFI. +; Check the low end of the AGFI range. define i64 @f4(i64 %dummy, ptr %src) { ; CHECK-LABEL: f4: -; CHECK: agfi %r0, -32769 +; CHECK: agfi %r0, -2147483648 ; CHECK: br %r14 - %res = atomicrmw sub ptr %src, i64 32769 seq_cst + %res = atomicrmw sub ptr %src, i64 2147483648 seq_cst ret i64 %res } -; Check the low end of the AGFI range. +; Check the next value up, which uses an SLGFI. define i64 @f5(i64 %dummy, ptr %src) { ; CHECK-LABEL: f5: -; CHECK: agfi %r0, -2147483648 +; CHECK: slgfi ; CHECK: br %r14 - %res = atomicrmw sub ptr %src, i64 2147483648 seq_cst + %res = atomicrmw sub ptr %src, i64 2147483649 seq_cst ret i64 %res } -; Check the next value up, which must use a register operation. +; Check subtraction of -1, which can use LA. define i64 @f6(i64 %dummy, ptr %src) { ; CHECK-LABEL: f6: -; CHECK: sgr +; CHECK: la %r0, 1(%r2) ; CHECK: br %r14 - %res = atomicrmw sub ptr %src, i64 2147483649 seq_cst + %res = atomicrmw sub ptr %src, i64 -1 seq_cst ret i64 %res } -; Check subtraction of -1, which can use AGHI. +; Check use of LAY. define i64 @f7(i64 %dummy, ptr %src) { ; CHECK-LABEL: f7: -; CHECK: aghi %r0, 1 -; CHECK: br %r14 - %res = atomicrmw sub ptr %src, i64 -1 seq_cst - ret i64 %res -} - -; Check the high end of the AGHI range. -define i64 @f8(i64 %dummy, ptr %src) { -; CHECK-LABEL: f8: -; CHECK: aghi %r0, 32767 +; CHECK: lay %r0, 32767(%r2) ; CHECK: br %r14 %res = atomicrmw sub ptr %src, i64 -32767 seq_cst ret i64 %res } -; Check the next value down, which must use AGFI instead. -define i64 @f9(i64 %dummy, ptr %src) { -; CHECK-LABEL: f9: -; CHECK: agfi %r0, 32768 -; CHECK: br %r14 - %res = atomicrmw sub ptr %src, i64 -32768 seq_cst - ret i64 %res -} - ; Check the high end of the AGFI range. -define i64 @f10(i64 %dummy, ptr %src) { -; CHECK-LABEL: f10: +define i64 @f8(i64 %dummy, ptr %src) { +; CHECK-LABEL: f8: ; CHECK: agfi %r0, 2147483647 ; CHECK: br %r14 %res = atomicrmw sub ptr %src, i64 -2147483647 seq_cst ret i64 %res } -; Check the next value down, which must use a register operation. -define i64 @f11(i64 %dummy, ptr %src) { -; CHECK-LABEL: f11: -; CHECK: sgr +; Check the next value down, which must use an ALGFI. +define i64 @f9(i64 %dummy, ptr %src) { +; CHECK-LABEL: f9: +; CHECK: algfi ; CHECK: br %r14 %res = atomicrmw sub ptr %src, i64 -2147483648 seq_cst ret i64 %res diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-03.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-03.ll index cf366b9f2e29..d99199724508 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-03.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-03.ll @@ -96,8 +96,8 @@ define i32 @f8(i32 %dummy, ptr %src, i32 %b) { ; Check that indexed addresses are not allowed. define i32 @f9(i32 %dummy, i64 %base, i64 %index, i32 %b) { ; CHECK-LABEL: f9: +; CHECK: l %r2, 0(%r4,%r3) ; CHECK: agr %r3, %r4 -; CHECK: l %r2, 0(%r3) ; CHECK: cs %r2, {{%r[0-9]+}}, 0(%r3) ; CHECK: br %r14 %add = add i64 %base, %index diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-04.ll index 9a493cb7fd8c..4797c5cf0679 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xchg-04.ll @@ -63,8 +63,8 @@ define i64 @f5(i64 %dummy, ptr %src, i64 %b) { ; Check that indexed addresses are not allowed. define i64 @f6(i64 %dummy, i64 %base, i64 %index, i64 %b) { ; CHECK-LABEL: f6: +; CHECK: lg %r2, 0(%r4,%r3) ; CHECK: agr %r3, %r4 -; CHECK: lg %r2, 0(%r3) ; CHECK: csg %r2, {{%r[0-9]+}}, 0(%r3) ; CHECK: br %r14 %add = add i64 %base, %index diff --git a/llvm/test/CodeGen/SystemZ/atomicrmw-xor-04.ll b/llvm/test/CodeGen/SystemZ/atomicrmw-xor-04.ll index 6cf1b80b8d0c..ee5fc1cf415d 100644 --- a/llvm/test/CodeGen/SystemZ/atomicrmw-xor-04.ll +++ b/llvm/test/CodeGen/SystemZ/atomicrmw-xor-04.ll @@ -48,11 +48,11 @@ define i64 @f4(i64 %dummy, ptr %src) { ret i64 %res } -; Check the next value up, which must use a register. (We could use -; combinations of XIH* and XIL* instead, but that isn't implemented.) +; Check the next value up. define i64 @f5(i64 %dummy, ptr %src) { ; CHECK-LABEL: f5: -; CHECK: xgr +; CHECK: xihf %r0, 1 +; CHECK: xilf %r0, 1 ; CHECK: br %r14 %res = atomicrmw xor ptr %src, i64 4294967297 seq_cst ret i64 %res @@ -70,7 +70,8 @@ define i64 @f6(i64 %dummy, ptr %src) { ; Check the next value up, which must use a register. define i64 @f7(i64 %dummy, ptr %src) { ; CHECK-LABEL: f7: -; CHECK: xgr +; CHECK: xihf %r0, 4294967295 +; CHECK: xilf %r0, 1 ; CHECK: br %r14 %res = atomicrmw xor ptr %src, i64 -4294967295 seq_cst ret i64 %res diff --git a/llvm/test/CodeGen/SystemZ/xor-04.ll b/llvm/test/CodeGen/SystemZ/xor-04.ll index 44f0a4cc39d0..ce5b76507b0d 100644 --- a/llvm/test/CodeGen/SystemZ/xor-04.ll +++ b/llvm/test/CodeGen/SystemZ/xor-04.ll @@ -61,8 +61,8 @@ define i64 @f6(i64 %a) { ; Check full bitwise negation define i64 @f7(i64 %a) { ; CHECK-LABEL: f7: -; CHECK: xihf %r2, 4294967295 -; CHECK: xilf %r2, 4294967295 +; CHECK: lcgr %r2, %r2 +; CHECK: aghi %r2, -1 ; CHECK: br %r14 %xor = xor i64 %a, -1 ret i64 %xor diff --git a/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll b/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll index cc7b5a7f3f81..7087041e8dac 100644 --- a/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll +++ b/llvm/test/CodeGen/Thumb2/mve-blockplacement.ll @@ -385,19 +385,17 @@ define i32 @d(i64 %e, i32 %f, i64 %g, i32 %h) { ; CHECK-NEXT: @ implicit-def: $r11 ; CHECK-NEXT: mov.w r9, #12 ; CHECK-NEXT: str r4, [sp, #12] @ 4-byte Spill -; CHECK-NEXT: add.w r0, r0, r2, lsr #1 ; CHECK-NEXT: add.w r1, r1, r2, lsr #1 -; CHECK-NEXT: movw r2, #65532 -; CHECK-NEXT: vdup.32 q6, r0 -; CHECK-NEXT: movt r2, #32767 -; CHECK-NEXT: and.w r3, r1, r2 +; CHECK-NEXT: add.w r0, r0, r2, lsr #1 +; CHECK-NEXT: bic r3, r1, #3 ; CHECK-NEXT: adr r1, .LCPI1_0 -; CHECK-NEXT: vdup.32 q7, r0 ; CHECK-NEXT: vldrw.u32 q0, [r1] ; CHECK-NEXT: adr r1, .LCPI1_1 ; CHECK-NEXT: vldrw.u32 q5, [r1] -; CHECK-NEXT: strd r3, r7, [sp, #4] @ 8-byte Folded Spill +; CHECK-NEXT: vdup.32 q6, r0 ; CHECK-NEXT: vadd.i32 q4, q0, r7 +; CHECK-NEXT: vdup.32 q7, r0 +; CHECK-NEXT: strd r3, r7, [sp, #4] @ 8-byte Folded Spill ; CHECK-NEXT: b .LBB1_6 ; CHECK-NEXT: .LBB1_2: @ %for.body6.preheader ; CHECK-NEXT: @ in Loop: Header=BB1_6 Depth=1 diff --git a/llvm/test/CodeGen/WebAssembly/wasm-eh-prepare.ll b/llvm/test/CodeGen/WebAssembly/wasm-eh-prepare.ll index e3de25101738..bd577e387c72 100644 --- a/llvm/test/CodeGen/WebAssembly/wasm-eh-prepare.ll +++ b/llvm/test/CodeGen/WebAssembly/wasm-eh-prepare.ll @@ -1,5 +1,5 @@ -; RUN: opt < %s -winehprepare -demote-catchswitch-only -wasm-eh-prepare -S | FileCheck %s -; RUN: opt < %s -winehprepare -demote-catchswitch-only -wasm-eh-prepare -S --mattr=+atomics,+bulk-memory | FileCheck %s +; RUN: opt < %s -win-eh-prepare -demote-catchswitch-only -wasm-eh-prepare -S | FileCheck %s +; RUN: opt < %s -win-eh-prepare -demote-catchswitch-only -wasm-eh-prepare -S --mattr=+atomics,+bulk-memory | FileCheck %s ; RUN: opt < %s -passes='win-eh-prepare,wasm-eh-prepare' -S | FileCheck %s ; RUN: opt < %s -passes='win-eh-prepare,wasm-eh-prepare' -S --mattr=+atomics,+bulk-memory | FileCheck %s diff --git a/llvm/test/CodeGen/WinEH/wineh-asm.ll b/llvm/test/CodeGen/WinEH/wineh-asm.ll index 00bb6ccb0a26..7aa29bb2be45 100644 --- a/llvm/test/CodeGen/WinEH/wineh-asm.ll +++ b/llvm/test/CodeGen/WinEH/wineh-asm.ll @@ -1,4 +1,4 @@ -; RUN: opt -winehprepare < %s +; RUN: opt -win-eh-prepare < %s ; RUN: opt -passes=win-eh-prepare < %s target triple = "x86_64-pc-windows-msvc" diff --git a/llvm/test/CodeGen/WinEH/wineh-cloning.ll b/llvm/test/CodeGen/WinEH/wineh-cloning.ll index 9a006326fed6..5df2eb26ead9 100644 --- a/llvm/test/CodeGen/WinEH/wineh-cloning.ll +++ b/llvm/test/CodeGen/WinEH/wineh-cloning.ll @@ -1,4 +1,4 @@ -; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -winehprepare < %s | FileCheck %s +; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -win-eh-prepare < %s | FileCheck %s ; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -passes=win-eh-prepare < %s | FileCheck %s declare i32 @__CxxFrameHandler3(...) diff --git a/llvm/test/CodeGen/WinEH/wineh-demotion.ll b/llvm/test/CodeGen/WinEH/wineh-demotion.ll index 326bb2d5f874..36a21e29f9c3 100644 --- a/llvm/test/CodeGen/WinEH/wineh-demotion.ll +++ b/llvm/test/CodeGen/WinEH/wineh-demotion.ll @@ -1,4 +1,4 @@ -; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -winehprepare < %s | FileCheck %s +; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -win-eh-prepare < %s | FileCheck %s ; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -passes=win-eh-prepare < %s | FileCheck %s declare i32 @__CxxFrameHandler3(...) diff --git a/llvm/test/CodeGen/WinEH/wineh-no-demotion.ll b/llvm/test/CodeGen/WinEH/wineh-no-demotion.ll index eebaf6d1c5be..d4667db97cc1 100644 --- a/llvm/test/CodeGen/WinEH/wineh-no-demotion.ll +++ b/llvm/test/CodeGen/WinEH/wineh-no-demotion.ll @@ -1,4 +1,4 @@ -; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -winehprepare -disable-demotion -disable-cleanups < %s | FileCheck %s +; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -win-eh-prepare -disable-demotion -disable-cleanups < %s | FileCheck %s ; RUN: opt -mtriple=x86_64-pc-windows-msvc -S -passes=win-eh-prepare -disable-demotion -disable-cleanups < %s | FileCheck %s declare i32 @__CxxFrameHandler3(...) diff --git a/llvm/test/CodeGen/X86/2011-10-19-widen_vselect.ll b/llvm/test/CodeGen/X86/2011-10-19-widen_vselect.ll index e7f62b9dfc22..171e16e35fc2 100644 --- a/llvm/test/CodeGen/X86/2011-10-19-widen_vselect.ll +++ b/llvm/test/CodeGen/X86/2011-10-19-widen_vselect.ll @@ -50,13 +50,12 @@ define void @zero_test() { ; X86-LABEL: zero_test: ; X86: # %bb.0: # %entry ; X86-NEXT: xorps %xmm0, %xmm0 -; X86-NEXT: movlps %xmm0, (%eax) +; X86-NEXT: movsd %xmm0, (%eax) ; X86-NEXT: retl ; ; X64-LABEL: zero_test: ; X64: # %bb.0: # %entry -; X64-NEXT: xorps %xmm0, %xmm0 -; X64-NEXT: movlps %xmm0, (%rax) +; X64-NEXT: movq $0, (%rax) ; X64-NEXT: retq entry: %0 = select <2 x i1> undef, <2 x float> undef, <2 x float> zeroinitializer diff --git a/llvm/test/CodeGen/X86/2012-07-10-extload64.ll b/llvm/test/CodeGen/X86/2012-07-10-extload64.ll index b6ec3b34eb10..6db17251cd59 100644 --- a/llvm/test/CodeGen/X86/2012-07-10-extload64.ll +++ b/llvm/test/CodeGen/X86/2012-07-10-extload64.ll @@ -30,7 +30,7 @@ define void @store_64(ptr %ptr) { ; X86: # %bb.0: # %BB ; X86-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-NEXT: xorps %xmm0, %xmm0 -; X86-NEXT: movlps %xmm0, (%eax) +; X86-NEXT: movsd %xmm0, (%eax) ; X86-NEXT: retl ; ; X64-LABEL: store_64: diff --git a/llvm/test/CodeGen/X86/atomic-nocx16.ll b/llvm/test/CodeGen/X86/atomic-nocx16.ll new file mode 100644 index 000000000000..5677541242a2 --- /dev/null +++ b/llvm/test/CodeGen/X86/atomic-nocx16.ll @@ -0,0 +1,49 @@ +; RUN: llc < %s -mtriple=x86_64-- -verify-machineinstrs -mcpu=corei7 -mattr=-cx16 | FileCheck %s +; RUN: llc < %s -mtriple=i386-linux-gnu -verify-machineinstrs -mattr=cx16 | FileCheck -check-prefix=CHECK %s + +;; Verify that 128-bit atomics emit a libcall without cx16 +;; available. +;; +;; We test 32-bit mode with -mattr=cx16, because it should have no +;; effect for 32-bit mode. + +; CHECK-LABEL: test: +define void @test(ptr %a) nounwind { +entry: +; CHECK: __sync_val_compare_and_swap_16 + %0 = cmpxchg ptr %a, i128 1, i128 1 seq_cst seq_cst +; CHECK: __sync_lock_test_and_set_16 + %1 = atomicrmw xchg ptr %a, i128 1 seq_cst +; CHECK: __sync_fetch_and_add_16 + %2 = atomicrmw add ptr %a, i128 1 seq_cst +; CHECK: __sync_fetch_and_sub_16 + %3 = atomicrmw sub ptr %a, i128 1 seq_cst +; CHECK: __sync_fetch_and_and_16 + %4 = atomicrmw and ptr %a, i128 1 seq_cst +; CHECK: __sync_fetch_and_nand_16 + %5 = atomicrmw nand ptr %a, i128 1 seq_cst +; CHECK: __sync_fetch_and_or_16 + %6 = atomicrmw or ptr %a, i128 1 seq_cst +; CHECK: __sync_fetch_and_xor_16 + %7 = atomicrmw xor ptr %a, i128 1 seq_cst +; CHECK: __sync_val_compare_and_swap_16 + %8 = load atomic i128, ptr %a seq_cst, align 16 +; CHECK: __sync_lock_test_and_set_16 + store atomic i128 %8, ptr %a seq_cst, align 16 + ret void +} + +; CHECK-LABEL: test_fp: +define void @test_fp(fp128* %a) nounwind { +entry: +; CHECK: __sync_lock_test_and_set_16 + %0 = atomicrmw xchg fp128* %a, fp128 0xL00000000000000004000900000000000 seq_cst +; Currently fails to compile: +; %1 = atomicrmw fadd fp128* %a, fp128 0xL00000000000000004000900000000000 seq_cst +; %2 = atomicrmw fsub fp128* %a, fp128 0xL00000000000000004000900000000000 seq_cst +; CHECK: __sync_val_compare_and_swap_16 + %1 = load atomic fp128, fp128* %a seq_cst, align 16 +; CHECK: __sync_lock_test_and_set_16 + store atomic fp128 %1, fp128* %a seq_cst, align 16 + ret void +} diff --git a/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll b/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll index 6c4d112330f2..9555c45086d6 100644 --- a/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll +++ b/llvm/test/CodeGen/X86/atomic-non-integer-fp128.ll @@ -1,35 +1,139 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=-sse | FileCheck %s --check-prefix=X64-NOSSE -; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs | FileCheck %s --check-prefix=X64-SSE +; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=sse2,cx16 | FileCheck %s --check-prefixes=X64-SSE +; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=avx,cx16 | FileCheck %s --check-prefixes=X64-AVX +; RUN: llc < %s -mtriple=x86_64-linux-generic -verify-machineinstrs -mattr=avx512f,cx16 | FileCheck %s --check-prefixes=X64-AVX -; Note: This test is testing that the lowering for atomics matches what we -; currently emit for non-atomics + the atomic restriction. The presence of -; particular lowering detail in these tests should not be read as requiring -; that detail for correctness unless it's related to the atomicity itself. -; (Specifically, there were reviewer questions about the lowering for halfs -; and their calling convention which remain unresolved.) +; Codegen of fp128 without cx16 is tested in atomic-nocx16.ll define void @store_fp128(ptr %fptr, fp128 %v) { -; X64-NOSSE-LABEL: store_fp128: -; X64-NOSSE: # %bb.0: -; X64-NOSSE-NEXT: pushq %rax -; X64-NOSSE-NEXT: .cfi_def_cfa_offset 16 -; X64-NOSSE-NEXT: callq __sync_lock_test_and_set_16@PLT -; X64-NOSSE-NEXT: popq %rax -; X64-NOSSE-NEXT: .cfi_def_cfa_offset 8 -; X64-NOSSE-NEXT: retq -; ; X64-SSE-LABEL: store_fp128: ; X64-SSE: # %bb.0: -; X64-SSE-NEXT: subq $24, %rsp -; X64-SSE-NEXT: .cfi_def_cfa_offset 32 -; X64-SSE-NEXT: movaps %xmm0, (%rsp) -; X64-SSE-NEXT: movq (%rsp), %rsi -; X64-SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx -; X64-SSE-NEXT: callq __sync_lock_test_and_set_16@PLT -; X64-SSE-NEXT: addq $24, %rsp +; X64-SSE-NEXT: pushq %rbx +; X64-SSE-NEXT: .cfi_def_cfa_offset 16 +; X64-SSE-NEXT: .cfi_offset %rbx, -16 +; X64-SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) +; X64-SSE-NEXT: movq -{{[0-9]+}}(%rsp), %rbx +; X64-SSE-NEXT: movq -{{[0-9]+}}(%rsp), %rcx +; X64-SSE-NEXT: movq (%rdi), %rax +; X64-SSE-NEXT: movq 8(%rdi), %rdx +; X64-SSE-NEXT: .p2align 4, 0x90 +; X64-SSE-NEXT: .LBB0_1: # %atomicrmw.start +; X64-SSE-NEXT: # =>This Inner Loop Header: Depth=1 +; X64-SSE-NEXT: lock cmpxchg16b (%rdi) +; X64-SSE-NEXT: jne .LBB0_1 +; X64-SSE-NEXT: # %bb.2: # %atomicrmw.end +; X64-SSE-NEXT: popq %rbx ; X64-SSE-NEXT: .cfi_def_cfa_offset 8 ; X64-SSE-NEXT: retq +; +; X64-AVX-LABEL: store_fp128: +; X64-AVX: # %bb.0: +; X64-AVX-NEXT: pushq %rbx +; X64-AVX-NEXT: .cfi_def_cfa_offset 16 +; X64-AVX-NEXT: .cfi_offset %rbx, -16 +; X64-AVX-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp) +; X64-AVX-NEXT: movq -{{[0-9]+}}(%rsp), %rbx +; X64-AVX-NEXT: movq -{{[0-9]+}}(%rsp), %rcx +; X64-AVX-NEXT: movq (%rdi), %rax +; X64-AVX-NEXT: movq 8(%rdi), %rdx +; X64-AVX-NEXT: .p2align 4, 0x90 +; X64-AVX-NEXT: .LBB0_1: # %atomicrmw.start +; X64-AVX-NEXT: # =>This Inner Loop Header: Depth=1 +; X64-AVX-NEXT: lock cmpxchg16b (%rdi) +; X64-AVX-NEXT: jne .LBB0_1 +; X64-AVX-NEXT: # %bb.2: # %atomicrmw.end +; X64-AVX-NEXT: popq %rbx +; X64-AVX-NEXT: .cfi_def_cfa_offset 8 +; X64-AVX-NEXT: retq store atomic fp128 %v, ptr %fptr unordered, align 16 ret void } + +define fp128 @load_fp128(ptr %fptr) { +; X64-SSE-LABEL: load_fp128: +; X64-SSE: # %bb.0: +; X64-SSE-NEXT: pushq %rbx +; X64-SSE-NEXT: .cfi_def_cfa_offset 16 +; X64-SSE-NEXT: .cfi_offset %rbx, -16 +; X64-SSE-NEXT: xorl %eax, %eax +; X64-SSE-NEXT: xorl %edx, %edx +; X64-SSE-NEXT: xorl %ecx, %ecx +; X64-SSE-NEXT: xorl %ebx, %ebx +; X64-SSE-NEXT: lock cmpxchg16b (%rdi) +; X64-SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp) +; X64-SSE-NEXT: movq %rax, -{{[0-9]+}}(%rsp) +; X64-SSE-NEXT: movaps -{{[0-9]+}}(%rsp), %xmm0 +; X64-SSE-NEXT: popq %rbx +; X64-SSE-NEXT: .cfi_def_cfa_offset 8 +; X64-SSE-NEXT: retq +; +; X64-AVX-LABEL: load_fp128: +; X64-AVX: # %bb.0: +; X64-AVX-NEXT: pushq %rbx +; X64-AVX-NEXT: .cfi_def_cfa_offset 16 +; X64-AVX-NEXT: .cfi_offset %rbx, -16 +; X64-AVX-NEXT: xorl %eax, %eax +; X64-AVX-NEXT: xorl %edx, %edx +; X64-AVX-NEXT: xorl %ecx, %ecx +; X64-AVX-NEXT: xorl %ebx, %ebx +; X64-AVX-NEXT: lock cmpxchg16b (%rdi) +; X64-AVX-NEXT: movq %rdx, -{{[0-9]+}}(%rsp) +; X64-AVX-NEXT: movq %rax, -{{[0-9]+}}(%rsp) +; X64-AVX-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0 +; X64-AVX-NEXT: popq %rbx +; X64-AVX-NEXT: .cfi_def_cfa_offset 8 +; X64-AVX-NEXT: retq + %v = load atomic fp128, ptr %fptr unordered, align 16 + ret fp128 %v +} + +define fp128 @exchange_fp128(ptr %fptr, fp128 %x) { +; X64-SSE-LABEL: exchange_fp128: +; X64-SSE: # %bb.0: +; X64-SSE-NEXT: pushq %rbx +; X64-SSE-NEXT: .cfi_def_cfa_offset 16 +; X64-SSE-NEXT: .cfi_offset %rbx, -16 +; X64-SSE-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) +; X64-SSE-NEXT: movq -{{[0-9]+}}(%rsp), %rbx +; X64-SSE-NEXT: movq -{{[0-9]+}}(%rsp), %rcx +; X64-SSE-NEXT: movq (%rdi), %rax +; X64-SSE-NEXT: movq 8(%rdi), %rdx +; X64-SSE-NEXT: .p2align 4, 0x90 +; X64-SSE-NEXT: .LBB2_1: # %atomicrmw.start +; X64-SSE-NEXT: # =>This Inner Loop Header: Depth=1 +; X64-SSE-NEXT: lock cmpxchg16b (%rdi) +; X64-SSE-NEXT: jne .LBB2_1 +; X64-SSE-NEXT: # %bb.2: # %atomicrmw.end +; X64-SSE-NEXT: movq %rax, -{{[0-9]+}}(%rsp) +; X64-SSE-NEXT: movq %rdx, -{{[0-9]+}}(%rsp) +; X64-SSE-NEXT: movaps -{{[0-9]+}}(%rsp), %xmm0 +; X64-SSE-NEXT: popq %rbx +; X64-SSE-NEXT: .cfi_def_cfa_offset 8 +; X64-SSE-NEXT: retq +; +; X64-AVX-LABEL: exchange_fp128: +; X64-AVX: # %bb.0: +; X64-AVX-NEXT: pushq %rbx +; X64-AVX-NEXT: .cfi_def_cfa_offset 16 +; X64-AVX-NEXT: .cfi_offset %rbx, -16 +; X64-AVX-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp) +; X64-AVX-NEXT: movq -{{[0-9]+}}(%rsp), %rbx +; X64-AVX-NEXT: movq -{{[0-9]+}}(%rsp), %rcx +; X64-AVX-NEXT: movq (%rdi), %rax +; X64-AVX-NEXT: movq 8(%rdi), %rdx +; X64-AVX-NEXT: .p2align 4, 0x90 +; X64-AVX-NEXT: .LBB2_1: # %atomicrmw.start +; X64-AVX-NEXT: # =>This Inner Loop Header: Depth=1 +; X64-AVX-NEXT: lock cmpxchg16b (%rdi) +; X64-AVX-NEXT: jne .LBB2_1 +; X64-AVX-NEXT: # %bb.2: # %atomicrmw.end +; X64-AVX-NEXT: movq %rax, -{{[0-9]+}}(%rsp) +; X64-AVX-NEXT: movq %rdx, -{{[0-9]+}}(%rsp) +; X64-AVX-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0 +; X64-AVX-NEXT: popq %rbx +; X64-AVX-NEXT: .cfi_def_cfa_offset 8 +; X64-AVX-NEXT: retq + %v = atomicrmw xchg ptr %fptr, fp128 %x monotonic, align 16 + ret fp128 %v +} + diff --git a/llvm/test/CodeGen/X86/atomic-non-integer.ll b/llvm/test/CodeGen/X86/atomic-non-integer.ll index 7d2810e57a25..9e6f584d8311 100644 --- a/llvm/test/CodeGen/X86/atomic-non-integer.ll +++ b/llvm/test/CodeGen/X86/atomic-non-integer.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs -mattr=sse | FileCheck %s --check-prefixes=X86,X86-SSE,X86-SSE1 -; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck %s --check-prefixes=X86,X86-SSE,X86-SSE2 +; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs -mattr=sse | FileCheck %s --check-prefixes=X86,X86-SSE1 +; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs -mattr=sse2 | FileCheck %s --check-prefixes=X86,X86-SSE2 ; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs -mattr=avx | FileCheck %s --check-prefixes=X86,X86-AVX ; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs -mattr=avx512f | FileCheck %s --check-prefixes=X86,X86-AVX ; RUN: llc < %s -mtriple=i386-linux-generic -verify-machineinstrs | FileCheck %s --check-prefixes=X86,X86-NOSSE @@ -131,94 +131,6 @@ define void @store_double(ptr %fptr, double %v) { ret void } -define void @store_fp128(ptr %fptr, fp128 %v) { -; X86-SSE-LABEL: store_fp128: -; X86-SSE: # %bb.0: -; X86-SSE-NEXT: subl $36, %esp -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 36 -; X86-SSE-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-SSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE-NEXT: pushl %eax -; X86-SSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE-NEXT: calll __sync_lock_test_and_set_16 -; X86-SSE-NEXT: .cfi_adjust_cfa_offset -4 -; X86-SSE-NEXT: addl $56, %esp -; X86-SSE-NEXT: .cfi_adjust_cfa_offset -56 -; X86-SSE-NEXT: retl -; -; X86-AVX-LABEL: store_fp128: -; X86-AVX: # %bb.0: -; X86-AVX-NEXT: subl $60, %esp -; X86-AVX-NEXT: .cfi_def_cfa_offset 64 -; X86-AVX-NEXT: vmovaps {{[0-9]+}}(%esp), %xmm0 -; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-AVX-NEXT: movl %eax, {{[0-9]+}}(%esp) -; X86-AVX-NEXT: vmovups %xmm0, {{[0-9]+}}(%esp) -; X86-AVX-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-AVX-NEXT: movl %eax, (%esp) -; X86-AVX-NEXT: calll __sync_lock_test_and_set_16 -; X86-AVX-NEXT: addl $56, %esp -; X86-AVX-NEXT: .cfi_def_cfa_offset 4 -; X86-AVX-NEXT: retl -; -; X86-NOSSE-LABEL: store_fp128: -; X86-NOSSE: # %bb.0: -; X86-NOSSE-NEXT: subl $36, %esp -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 36 -; X86-NOSSE-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-NOSSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl %eax -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: calll __sync_lock_test_and_set_16 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset -4 -; X86-NOSSE-NEXT: addl $56, %esp -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset -56 -; X86-NOSSE-NEXT: retl -; -; X64-SSE-LABEL: store_fp128: -; X64-SSE: # %bb.0: -; X64-SSE-NEXT: subq $24, %rsp -; X64-SSE-NEXT: .cfi_def_cfa_offset 32 -; X64-SSE-NEXT: movaps %xmm0, (%rsp) -; X64-SSE-NEXT: movq (%rsp), %rsi -; X64-SSE-NEXT: movq {{[0-9]+}}(%rsp), %rdx -; X64-SSE-NEXT: callq __sync_lock_test_and_set_16@PLT -; X64-SSE-NEXT: addq $24, %rsp -; X64-SSE-NEXT: .cfi_def_cfa_offset 8 -; X64-SSE-NEXT: retq -; -; X64-AVX-LABEL: store_fp128: -; X64-AVX: # %bb.0: -; X64-AVX-NEXT: subq $24, %rsp -; X64-AVX-NEXT: .cfi_def_cfa_offset 32 -; X64-AVX-NEXT: vmovaps %xmm0, (%rsp) -; X64-AVX-NEXT: movq (%rsp), %rsi -; X64-AVX-NEXT: movq {{[0-9]+}}(%rsp), %rdx -; X64-AVX-NEXT: callq __sync_lock_test_and_set_16@PLT -; X64-AVX-NEXT: addq $24, %rsp -; X64-AVX-NEXT: .cfi_def_cfa_offset 8 -; X64-AVX-NEXT: retq - store atomic fp128 %v, ptr %fptr unordered, align 16 - ret void -} define half @load_half(ptr %fptr) { ; X86-SSE1-LABEL: load_half: @@ -393,220 +305,273 @@ define double @load_double(ptr %fptr) { ret double %v } -define fp128 @load_fp128(ptr %fptr) { -; X86-SSE1-LABEL: load_fp128: +define half @exchange_half(ptr %fptr, half %x) { +; X86-SSE1-LABEL: exchange_half: +; X86-SSE1: # %bb.0: +; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE1-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; X86-SSE1-NEXT: xchgw %ax, (%ecx) +; X86-SSE1-NEXT: retl +; +; X86-SSE2-LABEL: exchange_half: +; X86-SSE2: # %bb.0: +; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-SSE2-NEXT: movzwl {{[0-9]+}}(%esp), %ecx +; X86-SSE2-NEXT: xchgw %cx, (%eax) +; X86-SSE2-NEXT: pinsrw $0, %ecx, %xmm0 +; X86-SSE2-NEXT: retl +; +; X86-AVX-LABEL: exchange_half: +; X86-AVX: # %bb.0: +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-AVX-NEXT: movzwl {{[0-9]+}}(%esp), %ecx +; X86-AVX-NEXT: xchgw %cx, (%eax) +; X86-AVX-NEXT: vpinsrw $0, %ecx, %xmm0, %xmm0 +; X86-AVX-NEXT: retl +; +; X86-NOSSE-LABEL: exchange_half: +; X86-NOSSE: # %bb.0: +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NOSSE-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; X86-NOSSE-NEXT: xchgw %ax, (%ecx) +; X86-NOSSE-NEXT: retl +; +; X64-SSE-LABEL: exchange_half: +; X64-SSE: # %bb.0: +; X64-SSE-NEXT: pextrw $0, %xmm0, %eax +; X64-SSE-NEXT: xchgw %ax, (%rdi) +; X64-SSE-NEXT: pinsrw $0, %eax, %xmm0 +; X64-SSE-NEXT: retq +; +; X64-AVX-LABEL: exchange_half: +; X64-AVX: # %bb.0: +; X64-AVX-NEXT: vpextrw $0, %xmm0, %eax +; X64-AVX-NEXT: xchgw %ax, (%rdi) +; X64-AVX-NEXT: vpinsrw $0, %eax, %xmm0, %xmm0 +; X64-AVX-NEXT: retq + %v = atomicrmw xchg ptr %fptr, half %x monotonic, align 2 + ret half %v +} + +define float @exchange_float(ptr %fptr, float %x) { +; X86-SSE1-LABEL: exchange_float: ; X86-SSE1: # %bb.0: -; X86-SSE1-NEXT: pushl %edi +; X86-SSE1-NEXT: pushl %eax +; X86-SSE1-NEXT: .cfi_def_cfa_offset 8 +; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE1-NEXT: xchgl %ecx, (%eax) +; X86-SSE1-NEXT: movl %ecx, (%esp) +; X86-SSE1-NEXT: flds (%esp) +; X86-SSE1-NEXT: popl %eax +; X86-SSE1-NEXT: .cfi_def_cfa_offset 4 +; X86-SSE1-NEXT: retl +; +; X86-SSE2-LABEL: exchange_float: +; X86-SSE2: # %bb.0: +; X86-SSE2-NEXT: pushl %eax +; X86-SSE2-NEXT: .cfi_def_cfa_offset 8 +; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE2-NEXT: xchgl %ecx, (%eax) +; X86-SSE2-NEXT: movd %ecx, %xmm0 +; X86-SSE2-NEXT: movd %xmm0, (%esp) +; X86-SSE2-NEXT: flds (%esp) +; X86-SSE2-NEXT: popl %eax +; X86-SSE2-NEXT: .cfi_def_cfa_offset 4 +; X86-SSE2-NEXT: retl +; +; X86-AVX-LABEL: exchange_float: +; X86-AVX: # %bb.0: +; X86-AVX-NEXT: pushl %eax +; X86-AVX-NEXT: .cfi_def_cfa_offset 8 +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-AVX-NEXT: xchgl %ecx, (%eax) +; X86-AVX-NEXT: vmovd %ecx, %xmm0 +; X86-AVX-NEXT: vmovd %xmm0, (%esp) +; X86-AVX-NEXT: flds (%esp) +; X86-AVX-NEXT: popl %eax +; X86-AVX-NEXT: .cfi_def_cfa_offset 4 +; X86-AVX-NEXT: retl +; +; X86-NOSSE-LABEL: exchange_float: +; X86-NOSSE: # %bb.0: +; X86-NOSSE-NEXT: pushl %eax +; X86-NOSSE-NEXT: .cfi_def_cfa_offset 8 +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-NOSSE-NEXT: xchgl %ecx, (%eax) +; X86-NOSSE-NEXT: movl %ecx, (%esp) +; X86-NOSSE-NEXT: flds (%esp) +; X86-NOSSE-NEXT: popl %eax +; X86-NOSSE-NEXT: .cfi_def_cfa_offset 4 +; X86-NOSSE-NEXT: retl +; +; X64-SSE-LABEL: exchange_float: +; X64-SSE: # %bb.0: +; X64-SSE-NEXT: movd %xmm0, %eax +; X64-SSE-NEXT: xchgl %eax, (%rdi) +; X64-SSE-NEXT: movd %eax, %xmm0 +; X64-SSE-NEXT: retq +; +; X64-AVX-LABEL: exchange_float: +; X64-AVX: # %bb.0: +; X64-AVX-NEXT: vmovd %xmm0, %eax +; X64-AVX-NEXT: xchgl %eax, (%rdi) +; X64-AVX-NEXT: vmovd %eax, %xmm0 +; X64-AVX-NEXT: retq + %v = atomicrmw xchg ptr %fptr, float %x monotonic, align 4 + ret float %v +} + +define double @exchange_double(ptr %fptr, double %x) { +; X86-SSE1-LABEL: exchange_double: +; X86-SSE1: # %bb.0: +; X86-SSE1-NEXT: pushl %ebx ; X86-SSE1-NEXT: .cfi_def_cfa_offset 8 ; X86-SSE1-NEXT: pushl %esi ; X86-SSE1-NEXT: .cfi_def_cfa_offset 12 -; X86-SSE1-NEXT: subl $20, %esp -; X86-SSE1-NEXT: .cfi_def_cfa_offset 32 +; X86-SSE1-NEXT: subl $12, %esp +; X86-SSE1-NEXT: .cfi_def_cfa_offset 24 ; X86-SSE1-NEXT: .cfi_offset %esi, -12 -; X86-SSE1-NEXT: .cfi_offset %edi, -8 +; X86-SSE1-NEXT: .cfi_offset %ebx, -8 ; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-SSE1-NEXT: subl $8, %esp -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 8 -; X86-SSE1-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl $0 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: pushl %eax -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE1-NEXT: calll __sync_val_compare_and_swap_16 -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset -4 -; X86-SSE1-NEXT: addl $44, %esp -; X86-SSE1-NEXT: .cfi_adjust_cfa_offset -44 -; X86-SSE1-NEXT: movl (%esp), %eax +; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %ebx ; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-SSE1-NEXT: movl {{[0-9]+}}(%esp), %edi -; X86-SSE1-NEXT: movl %edi, 8(%esi) -; X86-SSE1-NEXT: movl %edx, 12(%esi) -; X86-SSE1-NEXT: movl %eax, (%esi) -; X86-SSE1-NEXT: movl %ecx, 4(%esi) -; X86-SSE1-NEXT: movl %esi, %eax -; X86-SSE1-NEXT: addl $20, %esp +; X86-SSE1-NEXT: movl (%esi), %eax +; X86-SSE1-NEXT: movl 4(%esi), %edx +; X86-SSE1-NEXT: .p2align 4, 0x90 +; X86-SSE1-NEXT: .LBB8_1: # %atomicrmw.start +; X86-SSE1-NEXT: # =>This Inner Loop Header: Depth=1 +; X86-SSE1-NEXT: lock cmpxchg8b (%esi) +; X86-SSE1-NEXT: jne .LBB8_1 +; X86-SSE1-NEXT: # %bb.2: # %atomicrmw.end +; X86-SSE1-NEXT: movl %eax, (%esp) +; X86-SSE1-NEXT: movl %edx, {{[0-9]+}}(%esp) +; X86-SSE1-NEXT: fldl (%esp) +; X86-SSE1-NEXT: addl $12, %esp ; X86-SSE1-NEXT: .cfi_def_cfa_offset 12 ; X86-SSE1-NEXT: popl %esi ; X86-SSE1-NEXT: .cfi_def_cfa_offset 8 -; X86-SSE1-NEXT: popl %edi +; X86-SSE1-NEXT: popl %ebx ; X86-SSE1-NEXT: .cfi_def_cfa_offset 4 -; X86-SSE1-NEXT: retl $4 +; X86-SSE1-NEXT: retl ; -; X86-SSE2-LABEL: load_fp128: +; X86-SSE2-LABEL: exchange_double: ; X86-SSE2: # %bb.0: -; X86-SSE2-NEXT: pushl %esi +; X86-SSE2-NEXT: pushl %ebx ; X86-SSE2-NEXT: .cfi_def_cfa_offset 8 -; X86-SSE2-NEXT: subl $24, %esp -; X86-SSE2-NEXT: .cfi_def_cfa_offset 32 -; X86-SSE2-NEXT: .cfi_offset %esi, -8 +; X86-SSE2-NEXT: pushl %esi +; X86-SSE2-NEXT: .cfi_def_cfa_offset 12 +; X86-SSE2-NEXT: subl $12, %esp +; X86-SSE2-NEXT: .cfi_def_cfa_offset 24 +; X86-SSE2-NEXT: .cfi_offset %esi, -12 +; X86-SSE2-NEXT: .cfi_offset %ebx, -8 ; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-SSE2-NEXT: subl $8, %esp -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 8 -; X86-SSE2-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl $0 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl {{[0-9]+}}(%esp) -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: pushl %eax -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset 4 -; X86-SSE2-NEXT: calll __sync_val_compare_and_swap_16 -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset -4 -; X86-SSE2-NEXT: addl $44, %esp -; X86-SSE2-NEXT: .cfi_adjust_cfa_offset -44 -; X86-SSE2-NEXT: movaps (%esp), %xmm0 -; X86-SSE2-NEXT: movaps %xmm0, (%esi) -; X86-SSE2-NEXT: movl %esi, %eax -; X86-SSE2-NEXT: addl $24, %esp -; X86-SSE2-NEXT: .cfi_def_cfa_offset 8 +; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ebx +; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-SSE2-NEXT: movl (%esi), %eax +; X86-SSE2-NEXT: movl 4(%esi), %edx +; X86-SSE2-NEXT: .p2align 4, 0x90 +; X86-SSE2-NEXT: .LBB8_1: # %atomicrmw.start +; X86-SSE2-NEXT: # =>This Inner Loop Header: Depth=1 +; X86-SSE2-NEXT: lock cmpxchg8b (%esi) +; X86-SSE2-NEXT: jne .LBB8_1 +; X86-SSE2-NEXT: # %bb.2: # %atomicrmw.end +; X86-SSE2-NEXT: movd %eax, %xmm0 +; X86-SSE2-NEXT: movd %edx, %xmm1 +; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1] +; X86-SSE2-NEXT: movq %xmm0, (%esp) +; X86-SSE2-NEXT: fldl (%esp) +; X86-SSE2-NEXT: addl $12, %esp +; X86-SSE2-NEXT: .cfi_def_cfa_offset 12 ; X86-SSE2-NEXT: popl %esi +; X86-SSE2-NEXT: .cfi_def_cfa_offset 8 +; X86-SSE2-NEXT: popl %ebx ; X86-SSE2-NEXT: .cfi_def_cfa_offset 4 -; X86-SSE2-NEXT: retl $4 +; X86-SSE2-NEXT: retl ; -; X86-AVX-LABEL: load_fp128: +; X86-AVX-LABEL: exchange_double: ; X86-AVX: # %bb.0: -; X86-AVX-NEXT: pushl %esi +; X86-AVX-NEXT: pushl %ebx ; X86-AVX-NEXT: .cfi_def_cfa_offset 8 -; X86-AVX-NEXT: subl $72, %esp -; X86-AVX-NEXT: .cfi_def_cfa_offset 80 -; X86-AVX-NEXT: .cfi_offset %esi, -8 +; X86-AVX-NEXT: pushl %esi +; X86-AVX-NEXT: .cfi_def_cfa_offset 12 +; X86-AVX-NEXT: subl $12, %esp +; X86-AVX-NEXT: .cfi_def_cfa_offset 24 +; X86-AVX-NEXT: .cfi_offset %esi, -12 +; X86-AVX-NEXT: .cfi_offset %ebx, -8 ; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0 -; X86-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp) -; X86-AVX-NEXT: movl %eax, {{[0-9]+}}(%esp) -; X86-AVX-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-AVX-NEXT: movl %eax, (%esp) -; X86-AVX-NEXT: vzeroupper -; X86-AVX-NEXT: calll __sync_val_compare_and_swap_16 -; X86-AVX-NEXT: subl $4, %esp -; X86-AVX-NEXT: vmovaps {{[0-9]+}}(%esp), %xmm0 -; X86-AVX-NEXT: vmovaps %xmm0, (%esi) -; X86-AVX-NEXT: movl %esi, %eax -; X86-AVX-NEXT: addl $72, %esp -; X86-AVX-NEXT: .cfi_def_cfa_offset 8 +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %ebx +; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-AVX-NEXT: movl (%esi), %eax +; X86-AVX-NEXT: movl 4(%esi), %edx +; X86-AVX-NEXT: .p2align 4, 0x90 +; X86-AVX-NEXT: .LBB8_1: # %atomicrmw.start +; X86-AVX-NEXT: # =>This Inner Loop Header: Depth=1 +; X86-AVX-NEXT: lock cmpxchg8b (%esi) +; X86-AVX-NEXT: jne .LBB8_1 +; X86-AVX-NEXT: # %bb.2: # %atomicrmw.end +; X86-AVX-NEXT: vmovd %eax, %xmm0 +; X86-AVX-NEXT: vpinsrd $1, %edx, %xmm0, %xmm0 +; X86-AVX-NEXT: vmovq %xmm0, (%esp) +; X86-AVX-NEXT: fldl (%esp) +; X86-AVX-NEXT: addl $12, %esp +; X86-AVX-NEXT: .cfi_def_cfa_offset 12 ; X86-AVX-NEXT: popl %esi +; X86-AVX-NEXT: .cfi_def_cfa_offset 8 +; X86-AVX-NEXT: popl %ebx ; X86-AVX-NEXT: .cfi_def_cfa_offset 4 -; X86-AVX-NEXT: retl $4 +; X86-AVX-NEXT: retl ; -; X86-NOSSE-LABEL: load_fp128: +; X86-NOSSE-LABEL: exchange_double: ; X86-NOSSE: # %bb.0: -; X86-NOSSE-NEXT: pushl %edi +; X86-NOSSE-NEXT: pushl %ebx ; X86-NOSSE-NEXT: .cfi_def_cfa_offset 8 ; X86-NOSSE-NEXT: pushl %esi ; X86-NOSSE-NEXT: .cfi_def_cfa_offset 12 -; X86-NOSSE-NEXT: subl $20, %esp -; X86-NOSSE-NEXT: .cfi_def_cfa_offset 32 +; X86-NOSSE-NEXT: subl $12, %esp +; X86-NOSSE-NEXT: .cfi_def_cfa_offset 24 ; X86-NOSSE-NEXT: .cfi_offset %esi, -12 -; X86-NOSSE-NEXT: .cfi_offset %edi, -8 +; X86-NOSSE-NEXT: .cfi_offset %ebx, -8 ; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %esi -; X86-NOSSE-NEXT: subl $8, %esp -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 8 -; X86-NOSSE-NEXT: leal {{[0-9]+}}(%esp), %eax -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl $0 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl {{[0-9]+}}(%esp) -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: pushl %eax -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset 4 -; X86-NOSSE-NEXT: calll __sync_val_compare_and_swap_16 -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset -4 -; X86-NOSSE-NEXT: addl $44, %esp -; X86-NOSSE-NEXT: .cfi_adjust_cfa_offset -44 -; X86-NOSSE-NEXT: movl (%esp), %eax +; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ebx ; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %edx -; X86-NOSSE-NEXT: movl {{[0-9]+}}(%esp), %edi -; X86-NOSSE-NEXT: movl %edi, 8(%esi) -; X86-NOSSE-NEXT: movl %edx, 12(%esi) -; X86-NOSSE-NEXT: movl %eax, (%esi) -; X86-NOSSE-NEXT: movl %ecx, 4(%esi) -; X86-NOSSE-NEXT: movl %esi, %eax -; X86-NOSSE-NEXT: addl $20, %esp +; X86-NOSSE-NEXT: movl (%esi), %eax +; X86-NOSSE-NEXT: movl 4(%esi), %edx +; X86-NOSSE-NEXT: .p2align 4, 0x90 +; X86-NOSSE-NEXT: .LBB8_1: # %atomicrmw.start +; X86-NOSSE-NEXT: # =>This Inner Loop Header: Depth=1 +; X86-NOSSE-NEXT: lock cmpxchg8b (%esi) +; X86-NOSSE-NEXT: jne .LBB8_1 +; X86-NOSSE-NEXT: # %bb.2: # %atomicrmw.end +; X86-NOSSE-NEXT: movl %eax, (%esp) +; X86-NOSSE-NEXT: movl %edx, {{[0-9]+}}(%esp) +; X86-NOSSE-NEXT: fldl (%esp) +; X86-NOSSE-NEXT: addl $12, %esp ; X86-NOSSE-NEXT: .cfi_def_cfa_offset 12 ; X86-NOSSE-NEXT: popl %esi ; X86-NOSSE-NEXT: .cfi_def_cfa_offset 8 -; X86-NOSSE-NEXT: popl %edi +; X86-NOSSE-NEXT: popl %ebx ; X86-NOSSE-NEXT: .cfi_def_cfa_offset 4 -; X86-NOSSE-NEXT: retl $4 +; X86-NOSSE-NEXT: retl ; -; X64-SSE-LABEL: load_fp128: +; X64-SSE-LABEL: exchange_double: ; X64-SSE: # %bb.0: -; X64-SSE-NEXT: subq $24, %rsp -; X64-SSE-NEXT: .cfi_def_cfa_offset 32 -; X64-SSE-NEXT: xorl %esi, %esi -; X64-SSE-NEXT: xorl %edx, %edx -; X64-SSE-NEXT: xorl %ecx, %ecx -; X64-SSE-NEXT: xorl %r8d, %r8d -; X64-SSE-NEXT: callq __sync_val_compare_and_swap_16@PLT -; X64-SSE-NEXT: movq %rdx, {{[0-9]+}}(%rsp) -; X64-SSE-NEXT: movq %rax, (%rsp) -; X64-SSE-NEXT: movaps (%rsp), %xmm0 -; X64-SSE-NEXT: addq $24, %rsp -; X64-SSE-NEXT: .cfi_def_cfa_offset 8 +; X64-SSE-NEXT: movq %xmm0, %rax +; X64-SSE-NEXT: xchgq %rax, (%rdi) +; X64-SSE-NEXT: movq %rax, %xmm0 ; X64-SSE-NEXT: retq ; -; X64-AVX-LABEL: load_fp128: +; X64-AVX-LABEL: exchange_double: ; X64-AVX: # %bb.0: -; X64-AVX-NEXT: subq $24, %rsp -; X64-AVX-NEXT: .cfi_def_cfa_offset 32 -; X64-AVX-NEXT: xorl %esi, %esi -; X64-AVX-NEXT: xorl %edx, %edx -; X64-AVX-NEXT: xorl %ecx, %ecx -; X64-AVX-NEXT: xorl %r8d, %r8d -; X64-AVX-NEXT: callq __sync_val_compare_and_swap_16@PLT -; X64-AVX-NEXT: movq %rdx, {{[0-9]+}}(%rsp) -; X64-AVX-NEXT: movq %rax, (%rsp) -; X64-AVX-NEXT: vmovaps (%rsp), %xmm0 -; X64-AVX-NEXT: addq $24, %rsp -; X64-AVX-NEXT: .cfi_def_cfa_offset 8 +; X64-AVX-NEXT: vmovq %xmm0, %rax +; X64-AVX-NEXT: xchgq %rax, (%rdi) +; X64-AVX-NEXT: vmovq %rax, %xmm0 ; X64-AVX-NEXT: retq - %v = load atomic fp128, ptr %fptr unordered, align 16 - ret fp128 %v + %v = atomicrmw xchg ptr %fptr, double %x monotonic, align 8 + ret double %v } diff --git a/llvm/test/CodeGen/X86/atomic128.ll b/llvm/test/CodeGen/X86/atomic128.ll index d5600b54a169..1f7c2254bc79 100644 --- a/llvm/test/CodeGen/X86/atomic128.ll +++ b/llvm/test/CodeGen/X86/atomic128.ll @@ -1,7 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py ; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16 | FileCheck %s -; RUN: llc < %s -mtriple=i386-linux-gnu -verify-machineinstrs -mattr=cx16 | FileCheck %s -check-prefixes=CHECK32 -; RUN: llc < %s -mtriple=i386-linux-gnu -verify-machineinstrs -mattr=-cx16 | FileCheck %s -check-prefixes=CHECK32 + +; Codegen of i128 without cx16 is tested in atomic-nocx16.ll @var = global i128 0 @@ -20,61 +20,6 @@ define i128 @val_compare_and_swap(ptr %p, i128 %oldval, i128 %newval) { ; CHECK-NEXT: lock cmpxchg16b (%rdi) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: val_compare_and_swap: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %edi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 12 -; CHECK32-NEXT: subl $20, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -12 -; CHECK32-NEXT: .cfi_offset %edi, -8 -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_val_compare_and_swap_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $44, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -44 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edi -; CHECK32-NEXT: movl %edi, 8(%esi) -; CHECK32-NEXT: movl %edx, 12(%esi) -; CHECK32-NEXT: movl %eax, (%esi) -; CHECK32-NEXT: movl %ecx, 4(%esi) -; CHECK32-NEXT: movl %esi, %eax -; CHECK32-NEXT: addl $20, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 12 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %edi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl $4 %pair = cmpxchg ptr %p, i128 %oldval, i128 %newval acquire acquire %val = extractvalue { i128, i1 } %pair, 0 ret i128 %val @@ -94,24 +39,6 @@ define void @cmpxchg16b_global_with_offset() nounwind { ; CHECK-NEXT: lock cmpxchg16b _cmpxchg16b_global+16(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: cmpxchg16b_global_with_offset: -; CHECK32: # %bb.0: # %entry -; CHECK32-NEXT: subl $36, %esp -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: pushl $cmpxchg16b_global+16 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: calll __sync_val_compare_and_swap_16 -; CHECK32-NEXT: addl $72, %esp -; CHECK32-NEXT: retl entry: %0 = load atomic i128, ptr getelementptr inbounds ({i128, i128}, ptr @cmpxchg16b_global, i64 0, i32 1) acquire, align 16 ret void @@ -142,46 +69,6 @@ define void @fetch_and_nand(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_nand: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_nand_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw nand ptr %p, i128 %bits release store i128 %val, ptr @var, align 16 ret void @@ -210,46 +97,6 @@ define void @fetch_and_or(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_or: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_or_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw or ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -278,46 +125,6 @@ define void @fetch_and_add(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_add: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_add_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw add ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -346,46 +153,6 @@ define void @fetch_and_sub(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_sub: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_sub_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw sub ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -417,46 +184,6 @@ define void @fetch_and_min(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_min: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_min_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw min ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -488,46 +215,6 @@ define void @fetch_and_max(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_max: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_max_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw max ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -559,46 +246,6 @@ define void @fetch_and_umin(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_umin: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_umin_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw umin ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -630,46 +277,6 @@ define void @fetch_and_umax(ptr %p, i128 %bits) { ; CHECK-NEXT: movq %rdx, _var+8(%rip) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: fetch_and_umax: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: subl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -8 -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_fetch_and_umax_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $28, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -28 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: movl %esi, var+8 -; CHECK32-NEXT: movl %edx, var+12 -; CHECK32-NEXT: movl %eax, var -; CHECK32-NEXT: movl %ecx, var+4 -; CHECK32-NEXT: addl $24, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl %val = atomicrmw umax ptr %p, i128 %bits seq_cst store i128 %val, ptr @var, align 16 ret void @@ -688,61 +295,6 @@ define i128 @atomic_load_seq_cst(ptr %p) { ; CHECK-NEXT: lock cmpxchg16b (%rdi) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: atomic_load_seq_cst: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %edi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 12 -; CHECK32-NEXT: subl $20, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -12 -; CHECK32-NEXT: .cfi_offset %edi, -8 -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_val_compare_and_swap_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $44, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -44 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edi -; CHECK32-NEXT: movl %edi, 8(%esi) -; CHECK32-NEXT: movl %edx, 12(%esi) -; CHECK32-NEXT: movl %eax, (%esi) -; CHECK32-NEXT: movl %ecx, 4(%esi) -; CHECK32-NEXT: movl %esi, %eax -; CHECK32-NEXT: addl $20, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 12 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %edi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl $4 %r = load atomic i128, ptr %p seq_cst, align 16 ret i128 %r } @@ -760,61 +312,6 @@ define i128 @atomic_load_relaxed(ptr %p) { ; CHECK-NEXT: lock cmpxchg16b (%rdi) ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: atomic_load_relaxed: -; CHECK32: # %bb.0: -; CHECK32-NEXT: pushl %edi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: pushl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 12 -; CHECK32-NEXT: subl $20, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 32 -; CHECK32-NEXT: .cfi_offset %esi, -12 -; CHECK32-NEXT: .cfi_offset %edi, -8 -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %esi -; CHECK32-NEXT: subl $8, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 8 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl $0 -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_val_compare_and_swap_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $44, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -44 -; CHECK32-NEXT: movl (%esp), %eax -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edx -; CHECK32-NEXT: movl {{[0-9]+}}(%esp), %edi -; CHECK32-NEXT: movl %edi, 8(%esi) -; CHECK32-NEXT: movl %edx, 12(%esi) -; CHECK32-NEXT: movl %eax, (%esi) -; CHECK32-NEXT: movl %ecx, 4(%esi) -; CHECK32-NEXT: movl %esi, %eax -; CHECK32-NEXT: addl $20, %esp -; CHECK32-NEXT: .cfi_def_cfa_offset 12 -; CHECK32-NEXT: popl %esi -; CHECK32-NEXT: .cfi_def_cfa_offset 8 -; CHECK32-NEXT: popl %edi -; CHECK32-NEXT: .cfi_def_cfa_offset 4 -; CHECK32-NEXT: retl $4 %r = load atomic i128, ptr %p monotonic, align 16 ret i128 %r } @@ -837,29 +334,6 @@ define void @atomic_store_seq_cst(ptr %p, i128 %in) { ; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: atomic_store_seq_cst: -; CHECK32: # %bb.0: -; CHECK32-NEXT: subl $36, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 36 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_lock_test_and_set_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $56, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -56 -; CHECK32-NEXT: retl store atomic i128 %in, ptr %p seq_cst, align 16 ret void } @@ -882,29 +356,6 @@ define void @atomic_store_release(ptr %p, i128 %in) { ; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: atomic_store_release: -; CHECK32: # %bb.0: -; CHECK32-NEXT: subl $36, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 36 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_lock_test_and_set_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $56, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -56 -; CHECK32-NEXT: retl store atomic i128 %in, ptr %p release, align 16 ret void } @@ -927,29 +378,6 @@ define void @atomic_store_relaxed(ptr %p, i128 %in) { ; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end ; CHECK-NEXT: popq %rbx ; CHECK-NEXT: retq -; -; CHECK32-LABEL: atomic_store_relaxed: -; CHECK32: # %bb.0: -; CHECK32-NEXT: subl $36, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset 36 -; CHECK32-NEXT: leal {{[0-9]+}}(%esp), %eax -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl {{[0-9]+}}(%esp) -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: pushl %eax -; CHECK32-NEXT: .cfi_adjust_cfa_offset 4 -; CHECK32-NEXT: calll __sync_lock_test_and_set_16 -; CHECK32-NEXT: .cfi_adjust_cfa_offset -4 -; CHECK32-NEXT: addl $56, %esp -; CHECK32-NEXT: .cfi_adjust_cfa_offset -56 -; CHECK32-NEXT: retl store atomic i128 %in, ptr %p unordered, align 16 ret void } diff --git a/llvm/test/CodeGen/X86/atomicf128.ll b/llvm/test/CodeGen/X86/atomicf128.ll deleted file mode 100644 index 3b0bba403aa5..000000000000 --- a/llvm/test/CodeGen/X86/atomicf128.ll +++ /dev/null @@ -1,46 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16 | FileCheck %s -; RUN: llc < %s -mtriple=x86_64-apple-macosx10.9 -verify-machineinstrs -mattr=cx16 -mattr=-sse | FileCheck %s --check-prefix=NOSSE - -; FIXME: This test has a fatal error in 32-bit mode - -@fsc128 = external global fp128 - -define void @atomic_fetch_swapf128(fp128 %x) nounwind { -; CHECK-LABEL: atomic_fetch_swapf128: -; CHECK: ## %bb.0: -; CHECK-NEXT: pushq %rbx -; CHECK-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp) -; CHECK-NEXT: movq -{{[0-9]+}}(%rsp), %rbx -; CHECK-NEXT: movq -{{[0-9]+}}(%rsp), %rcx -; CHECK-NEXT: movq _fsc128@GOTPCREL(%rip), %rsi -; CHECK-NEXT: movq (%rsi), %rax -; CHECK-NEXT: movq 8(%rsi), %rdx -; CHECK-NEXT: .p2align 4, 0x90 -; CHECK-NEXT: LBB0_1: ## %atomicrmw.start -; CHECK-NEXT: ## =>This Inner Loop Header: Depth=1 -; CHECK-NEXT: lock cmpxchg16b (%rsi) -; CHECK-NEXT: jne LBB0_1 -; CHECK-NEXT: ## %bb.2: ## %atomicrmw.end -; CHECK-NEXT: popq %rbx -; CHECK-NEXT: retq -; -; NOSSE-LABEL: atomic_fetch_swapf128: -; NOSSE: ## %bb.0: -; NOSSE-NEXT: pushq %rbx -; NOSSE-NEXT: movq %rsi, %rcx -; NOSSE-NEXT: movq %rdi, %rbx -; NOSSE-NEXT: movq _fsc128@GOTPCREL(%rip), %rsi -; NOSSE-NEXT: movq (%rsi), %rax -; NOSSE-NEXT: movq 8(%rsi), %rdx -; NOSSE-NEXT: .p2align 4, 0x90 -; NOSSE-NEXT: LBB0_1: ## %atomicrmw.start -; NOSSE-NEXT: ## =>This Inner Loop Header: Depth=1 -; NOSSE-NEXT: lock cmpxchg16b (%rsi) -; NOSSE-NEXT: jne LBB0_1 -; NOSSE-NEXT: ## %bb.2: ## %atomicrmw.end -; NOSSE-NEXT: popq %rbx -; NOSSE-NEXT: retq - %t1 = atomicrmw xchg ptr @fsc128, fp128 %x acquire - ret void -} diff --git a/llvm/test/CodeGen/X86/avx512fp16-arith.ll b/llvm/test/CodeGen/X86/avx512fp16-arith.ll index 77cab3d1512c..8d811d8d29e0 100644 --- a/llvm/test/CodeGen/X86/avx512fp16-arith.ll +++ b/llvm/test/CodeGen/X86/avx512fp16-arith.ll @@ -329,7 +329,7 @@ define half @fcopysign(half %x, half %y) { ; CHECK-LABEL: fcopysign: ; CHECK: ## %bb.0: ; CHECK-NEXT: vpbroadcastw {{.*#+}} xmm2 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; CHECK-NEXT: vpternlogq $226, %xmm1, %xmm2, %xmm0 +; CHECK-NEXT: vpternlogd $226, %xmm1, %xmm2, %xmm0 ; CHECK-NEXT: retq %a = call half @llvm.copysign.f16(half %x, half %y) ret half %a @@ -384,7 +384,7 @@ declare <8 x half> @llvm.fabs.v8f16(<8 x half>) define <8 x half> @fcopysignv8f16(<8 x half> %x, <8 x half> %y) { ; CHECK-LABEL: fcopysignv8f16: ; CHECK: ## %bb.0: -; CHECK-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; CHECK-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; CHECK-NEXT: retq %a = call <8 x half> @llvm.copysign.v8f16(<8 x half> %x, <8 x half> %y) ret <8 x half> %a @@ -439,7 +439,7 @@ declare <16 x half> @llvm.fabs.v16f16(<16 x half>) define <16 x half> @fcopysignv16f16(<16 x half> %x, <16 x half> %y) { ; CHECK-LABEL: fcopysignv16f16: ; CHECK: ## %bb.0: -; CHECK-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; CHECK-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; CHECK-NEXT: retq %a = call <16 x half> @llvm.copysign.v16f16(<16 x half> %x, <16 x half> %y) ret <16 x half> %a @@ -494,7 +494,7 @@ declare <32 x half> @llvm.fabs.v32f16(<32 x half>) define <32 x half> @fcopysignv32f16(<32 x half> %x, <32 x half> %y) { ; CHECK-LABEL: fcopysignv32f16: ; CHECK: ## %bb.0: -; CHECK-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; CHECK-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; CHECK-NEXT: retq %a = call <32 x half> @llvm.copysign.v32f16(<32 x half> %x, <32 x half> %y) ret <32 x half> %a diff --git a/llvm/test/CodeGen/X86/code-model-elf.ll b/llvm/test/CodeGen/X86/code-model-elf.ll index 483ffd87ac69..457c6bde354f 100644 --- a/llvm/test/CodeGen/X86/code-model-elf.ll +++ b/llvm/test/CodeGen/X86/code-model-elf.ll @@ -41,7 +41,8 @@ target triple = "x86_64--linux" @extern_data = external global [10 x i32], align 16 @thread_data = external thread_local global i32, align 4 @unknown_size_data = dso_local global [0 x i32] zeroinitializer, align 16 - +@forced_small_data = dso_local global [10 x i32] zeroinitializer, code_model "small", align 16 +@forced_large_data = dso_local global [10 x i32] zeroinitializer, code_model "large", align 16 define dso_local ptr @lea_static_data() #0 { ; SMALL-STATIC-LABEL: lea_static_data: @@ -223,6 +224,199 @@ define dso_local ptr @lea_unknown_size_data() #0 { ret ptr @unknown_size_data } +; TODO: make nopic small and medium instruction sequence the same +define dso_local ptr @lea_forced_small_data() #0 { +; SMALL-STATIC-LABEL: lea_forced_small_data: +; SMALL-STATIC: # %bb.0: +; SMALL-STATIC-NEXT: movl $forced_small_data, %eax +; SMALL-STATIC-NEXT: retq +; +; MEDIUM-STATIC-LABEL: lea_forced_small_data: +; MEDIUM-STATIC: # %bb.0: +; MEDIUM-STATIC-NEXT: movabsq $forced_small_data, %rax +; MEDIUM-STATIC-NEXT: retq +; +; LARGE-STATIC-LABEL: lea_forced_small_data: +; LARGE-STATIC: # %bb.0: +; LARGE-STATIC-NEXT: movabsq $forced_small_data, %rax +; LARGE-STATIC-NEXT: retq +; +; SMALL-PIC-LABEL: lea_forced_small_data: +; SMALL-PIC: # %bb.0: +; SMALL-PIC-NEXT: leaq forced_small_data(%rip), %rax +; SMALL-PIC-NEXT: retq +; +; MEDIUM-SMALL-DATA-PIC-LABEL: lea_forced_small_data: +; MEDIUM-SMALL-DATA-PIC: # %bb.0: +; MEDIUM-SMALL-DATA-PIC-NEXT: leaq forced_small_data(%rip), %rax +; MEDIUM-SMALL-DATA-PIC-NEXT: retq +; +; MEDIUM-PIC-LABEL: lea_forced_small_data: +; MEDIUM-PIC: # %bb.0: +; MEDIUM-PIC-NEXT: leaq forced_small_data(%rip), %rax +; MEDIUM-PIC-NEXT: retq +; +; LARGE-PIC-LABEL: lea_forced_small_data: +; LARGE-PIC: # %bb.0: +; LARGE-PIC-NEXT: .L4$pb: +; LARGE-PIC-NEXT: leaq .L4$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L4$pb, %rcx +; LARGE-PIC-NEXT: addq %rax, %rcx +; LARGE-PIC-NEXT: movabsq $forced_small_data@GOTOFF, %rax +; LARGE-PIC-NEXT: addq %rcx, %rax +; LARGE-PIC-NEXT: retq + ret ptr @forced_small_data +} + +; TODO: make small and medium instruction sequence the same +define dso_local i32 @load_forced_small_data() #0 { +; SMALL-STATIC-LABEL: load_forced_small_data: +; SMALL-STATIC: # %bb.0: +; SMALL-STATIC-NEXT: movl forced_small_data+8(%rip), %eax +; SMALL-STATIC-NEXT: retq +; +; MEDIUM-STATIC-LABEL: load_forced_small_data: +; MEDIUM-STATIC: # %bb.0: +; MEDIUM-STATIC-NEXT: movabsq $forced_small_data, %rax +; MEDIUM-STATIC-NEXT: movl 8(%rax), %eax +; MEDIUM-STATIC-NEXT: retq +; +; LARGE-STATIC-LABEL: load_forced_small_data: +; LARGE-STATIC: # %bb.0: +; LARGE-STATIC-NEXT: movabsq $forced_small_data, %rax +; LARGE-STATIC-NEXT: movl 8(%rax), %eax +; LARGE-STATIC-NEXT: retq +; +; SMALL-PIC-LABEL: load_forced_small_data: +; SMALL-PIC: # %bb.0: +; SMALL-PIC-NEXT: movl forced_small_data+8(%rip), %eax +; SMALL-PIC-NEXT: retq +; +; MEDIUM-SMALL-DATA-PIC-LABEL: load_forced_small_data: +; MEDIUM-SMALL-DATA-PIC: # %bb.0: +; MEDIUM-SMALL-DATA-PIC-NEXT: leaq forced_small_data(%rip), %rax +; MEDIUM-SMALL-DATA-PIC-NEXT: movl 8(%rax), %eax +; MEDIUM-SMALL-DATA-PIC-NEXT: retq +; +; MEDIUM-PIC-LABEL: load_forced_small_data: +; MEDIUM-PIC: # %bb.0: +; MEDIUM-PIC-NEXT: leaq forced_small_data(%rip), %rax +; MEDIUM-PIC-NEXT: movl 8(%rax), %eax +; MEDIUM-PIC-NEXT: retq +; +; LARGE-PIC-LABEL: load_forced_small_data: +; LARGE-PIC: # %bb.0: +; LARGE-PIC-NEXT: .L5$pb: +; LARGE-PIC-NEXT: leaq .L5$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L5$pb, %rcx +; LARGE-PIC-NEXT: addq %rax, %rcx +; LARGE-PIC-NEXT: movabsq $forced_small_data@GOTOFF, %rax +; LARGE-PIC-NEXT: movl 8(%rcx,%rax), %eax +; LARGE-PIC-NEXT: retq + %rv = load i32, ptr getelementptr inbounds (i32, ptr @forced_small_data, i64 2) + ret i32 %rv +} + +; TODO: fix small code model instruction sequences to use 64-bit constants +define dso_local ptr @lea_forced_large_data() #0 { +; SMALL-STATIC-LABEL: lea_forced_large_data: +; SMALL-STATIC: # %bb.0: +; SMALL-STATIC-NEXT: movl $forced_large_data, %eax +; SMALL-STATIC-NEXT: retq +; +; MEDIUM-STATIC-LABEL: lea_forced_large_data: +; MEDIUM-STATIC: # %bb.0: +; MEDIUM-STATIC-NEXT: movabsq $forced_large_data, %rax +; MEDIUM-STATIC-NEXT: retq +; +; LARGE-STATIC-LABEL: lea_forced_large_data: +; LARGE-STATIC: # %bb.0: +; LARGE-STATIC-NEXT: movabsq $forced_large_data, %rax +; LARGE-STATIC-NEXT: retq +; +; SMALL-PIC-LABEL: lea_forced_large_data: +; SMALL-PIC: # %bb.0: +; SMALL-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax +; SMALL-PIC-NEXT: leaq forced_large_data@GOTOFF(%rax), %rax +; SMALL-PIC-NEXT: retq +; +; MEDIUM-SMALL-DATA-PIC-LABEL: lea_forced_large_data: +; MEDIUM-SMALL-DATA-PIC: # %bb.0: +; MEDIUM-SMALL-DATA-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rcx +; MEDIUM-SMALL-DATA-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rax +; MEDIUM-SMALL-DATA-PIC-NEXT: addq %rcx, %rax +; MEDIUM-SMALL-DATA-PIC-NEXT: retq +; +; MEDIUM-PIC-LABEL: lea_forced_large_data: +; MEDIUM-PIC: # %bb.0: +; MEDIUM-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rcx +; MEDIUM-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rax +; MEDIUM-PIC-NEXT: addq %rcx, %rax +; MEDIUM-PIC-NEXT: retq +; +; LARGE-PIC-LABEL: lea_forced_large_data: +; LARGE-PIC: # %bb.0: +; LARGE-PIC-NEXT: .L6$pb: +; LARGE-PIC-NEXT: leaq .L6$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L6$pb, %rcx +; LARGE-PIC-NEXT: addq %rax, %rcx +; LARGE-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rax +; LARGE-PIC-NEXT: addq %rcx, %rax +; LARGE-PIC-NEXT: retq + ret ptr @forced_large_data +} + +define dso_local i32 @load_forced_large_data() #0 { +; SMALL-STATIC-LABEL: load_forced_large_data: +; SMALL-STATIC: # %bb.0: +; SMALL-STATIC-NEXT: movl forced_large_data+8(%rip), %eax +; SMALL-STATIC-NEXT: retq +; +; MEDIUM-STATIC-LABEL: load_forced_large_data: +; MEDIUM-STATIC: # %bb.0: +; MEDIUM-STATIC-NEXT: movabsq $forced_large_data, %rax +; MEDIUM-STATIC-NEXT: movl 8(%rax), %eax +; MEDIUM-STATIC-NEXT: retq +; +; LARGE-STATIC-LABEL: load_forced_large_data: +; LARGE-STATIC: # %bb.0: +; LARGE-STATIC-NEXT: movabsq $forced_large_data, %rax +; LARGE-STATIC-NEXT: movl 8(%rax), %eax +; LARGE-STATIC-NEXT: retq +; +; SMALL-PIC-LABEL: load_forced_large_data: +; SMALL-PIC: # %bb.0: +; SMALL-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax +; SMALL-PIC-NEXT: movl forced_large_data@GOTOFF+8(%rax), %eax +; SMALL-PIC-NEXT: retq +; +; MEDIUM-SMALL-DATA-PIC-LABEL: load_forced_large_data: +; MEDIUM-SMALL-DATA-PIC: # %bb.0: +; MEDIUM-SMALL-DATA-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax +; MEDIUM-SMALL-DATA-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rcx +; MEDIUM-SMALL-DATA-PIC-NEXT: movl 8(%rax,%rcx), %eax +; MEDIUM-SMALL-DATA-PIC-NEXT: retq +; +; MEDIUM-PIC-LABEL: load_forced_large_data: +; MEDIUM-PIC: # %bb.0: +; MEDIUM-PIC-NEXT: leaq _GLOBAL_OFFSET_TABLE_(%rip), %rax +; MEDIUM-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rcx +; MEDIUM-PIC-NEXT: movl 8(%rax,%rcx), %eax +; MEDIUM-PIC-NEXT: retq +; +; LARGE-PIC-LABEL: load_forced_large_data: +; LARGE-PIC: # %bb.0: +; LARGE-PIC-NEXT: .L7$pb: +; LARGE-PIC-NEXT: leaq .L7$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L7$pb, %rcx +; LARGE-PIC-NEXT: addq %rax, %rcx +; LARGE-PIC-NEXT: movabsq $forced_large_data@GOTOFF, %rax +; LARGE-PIC-NEXT: movl 8(%rcx,%rax), %eax +; LARGE-PIC-NEXT: retq + %rv = load i32, ptr getelementptr inbounds (i32, ptr @forced_large_data, i64 2) + ret i32 %rv +} + define dso_local i32 @load_global_data() #0 { ; SMALL-STATIC-LABEL: load_global_data: ; SMALL-STATIC: # %bb.0: @@ -261,9 +455,9 @@ define dso_local i32 @load_global_data() #0 { ; ; LARGE-PIC-LABEL: load_global_data: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L4$pb: -; LARGE-PIC-NEXT: leaq .L4$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L4$pb, %rcx +; LARGE-PIC-NEXT: .L8$pb: +; LARGE-PIC-NEXT: leaq .L8$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L8$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $global_data@GOTOFF, %rax ; LARGE-PIC-NEXT: movl 8(%rcx,%rax), %eax @@ -311,9 +505,9 @@ define dso_local i32 @load_extern_data() #0 { ; ; LARGE-PIC-LABEL: load_extern_data: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L5$pb: -; LARGE-PIC-NEXT: leaq .L5$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L5$pb, %rcx +; LARGE-PIC-NEXT: .L9$pb: +; LARGE-PIC-NEXT: leaq .L9$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L9$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $extern_data@GOT, %rax ; LARGE-PIC-NEXT: movq (%rcx,%rax), %rax @@ -362,9 +556,9 @@ define dso_local i32 @load_unknown_size_data() #0 { ; ; LARGE-PIC-LABEL: load_unknown_size_data: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L6$pb: -; LARGE-PIC-NEXT: leaq .L6$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L6$pb, %rcx +; LARGE-PIC-NEXT: .L10$pb: +; LARGE-PIC-NEXT: leaq .L10$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L10$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $unknown_size_data@GOTOFF, %rax ; LARGE-PIC-NEXT: movl 8(%rcx,%rax), %eax @@ -422,9 +616,9 @@ define dso_local ptr @lea_static_fn() #0 { ; ; LARGE-PIC-LABEL: lea_static_fn: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L9$pb: -; LARGE-PIC-NEXT: leaq .L9$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L9$pb, %rcx +; LARGE-PIC-NEXT: .L13$pb: +; LARGE-PIC-NEXT: leaq .L13$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L13$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $static_fn@GOTOFF, %rax ; LARGE-PIC-NEXT: addq %rcx, %rax @@ -465,9 +659,9 @@ define dso_local ptr @lea_global_fn() #0 { ; ; LARGE-PIC-LABEL: lea_global_fn: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L10$pb: -; LARGE-PIC-NEXT: leaq .L10$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L10$pb, %rcx +; LARGE-PIC-NEXT: .L14$pb: +; LARGE-PIC-NEXT: leaq .L14$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L14$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $global_fn@GOTOFF, %rax ; LARGE-PIC-NEXT: addq %rcx, %rax @@ -508,9 +702,9 @@ define dso_local ptr @lea_extern_fn() #0 { ; ; LARGE-PIC-LABEL: lea_extern_fn: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L11$pb: -; LARGE-PIC-NEXT: leaq .L11$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L11$pb, %rcx +; LARGE-PIC-NEXT: .L15$pb: +; LARGE-PIC-NEXT: leaq .L15$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L15$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq $extern_fn@GOT, %rax ; LARGE-PIC-NEXT: movq (%rcx,%rax), %rax @@ -586,9 +780,9 @@ define dso_local float @load_constant_pool(float %x) #0 { ; ; LARGE-PIC-LABEL: load_constant_pool: ; LARGE-PIC: # %bb.0: -; LARGE-PIC-NEXT: .L13$pb: -; LARGE-PIC-NEXT: leaq .L13$pb(%rip), %rax -; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L13$pb, %rcx +; LARGE-PIC-NEXT: .L17$pb: +; LARGE-PIC-NEXT: leaq .L17$pb(%rip), %rax +; LARGE-PIC-NEXT: movabsq $_GLOBAL_OFFSET_TABLE_-.L17$pb, %rcx ; LARGE-PIC-NEXT: addq %rax, %rcx ; LARGE-PIC-NEXT: movabsq ${{\.?LCPI[0-9]+_[0-9]+}}@GOTOFF, %rax ; LARGE-PIC-NEXT: addss (%rcx,%rax), %xmm0 diff --git a/llvm/test/CodeGen/X86/combine-concatvectors.ll b/llvm/test/CodeGen/X86/combine-concatvectors.ll index a8c3fb589bbf..31eaa1b205aa 100644 --- a/llvm/test/CodeGen/X86/combine-concatvectors.ll +++ b/llvm/test/CodeGen/X86/combine-concatvectors.ll @@ -48,8 +48,7 @@ define void @concat_of_broadcast_v2f64_v4f64() { ; AVX1-NEXT: movl $1091567616, 30256(%rax) # imm = 0x41100000 ; AVX1-NEXT: movabsq $4294967297, %rcx # imm = 0x100000001 ; AVX1-NEXT: movq %rcx, 46348(%rax) -; AVX1-NEXT: vbroadcastf128 {{.*#+}} ymm0 = [7.812501848093234E-3,7.812501848093234E-3,7.812501848093234E-3,7.812501848093234E-3] -; AVX1-NEXT: # ymm0 = mem[0,1,0,1] +; AVX1-NEXT: vbroadcastss {{.*#+}} ymm0 = [1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0,1.0E+0] ; AVX1-NEXT: vmovups %ymm0, 48296(%rax) ; AVX1-NEXT: vmovsd {{.*#+}} xmm0 = mem[0],zero ; AVX1-NEXT: vmovsd %xmm0, 47372(%rax) diff --git a/llvm/test/CodeGen/X86/dwarf-eh-prepare-dbg.ll b/llvm/test/CodeGen/X86/dwarf-eh-prepare-dbg.ll index 651e196a4ed7..0345b736e9e6 100644 --- a/llvm/test/CodeGen/X86/dwarf-eh-prepare-dbg.ll +++ b/llvm/test/CodeGen/X86/dwarf-eh-prepare-dbg.ll @@ -1,6 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py -; RUN: opt -S -mtriple=x86_64-linux-gnu -dwarfehprepare < %s | FileCheck %s -; RUN: opt -S -mtriple=x86_64-linux-gnu -passes=dwarfehprepare < %s | FileCheck %s +; RUN: opt -S -mtriple=x86_64-linux-gnu -dwarf-eh-prepare < %s | FileCheck %s +; RUN: opt -S -mtriple=x86_64-linux-gnu -passes=dwarf-eh-prepare < %s | FileCheck %s ; PR57469: If _Unwind_Resume is defined in the same module and we have debug ; info, then the inserted _Unwind_Resume calls also need to have a dummy debug diff --git a/llvm/test/CodeGen/X86/dwarf-eh-prepare.ll b/llvm/test/CodeGen/X86/dwarf-eh-prepare.ll index 6778601cfbf6..dd27e7fe1411 100644 --- a/llvm/test/CodeGen/X86/dwarf-eh-prepare.ll +++ b/llvm/test/CodeGen/X86/dwarf-eh-prepare.ll @@ -1,5 +1,5 @@ -; RUN: opt -mtriple=x86_64-linux-gnu -dwarfehprepare -simplifycfg-require-and-preserve-domtree=1 -run-twice < %s -S | FileCheck %s -; RUN: opt -mtriple=x86_64-linux-gnu -passes=dwarfehprepare -codegen-opt-level=2 -simplifycfg-require-and-preserve-domtree=1 -run-twice < %s -S | FileCheck %s +; RUN: opt -mtriple=x86_64-linux-gnu -dwarf-eh-prepare -simplifycfg-require-and-preserve-domtree=1 -run-twice < %s -S | FileCheck %s +; RUN: opt -mtriple=x86_64-linux-gnu -passes=dwarf-eh-prepare -codegen-opt-level=2 -simplifycfg-require-and-preserve-domtree=1 -run-twice < %s -S | FileCheck %s ; Check basic functionality of IR-to-IR DWARF EH preparation. This should ; eliminate resumes. This pass requires a TargetMachine, so we put it under X86 diff --git a/llvm/test/CodeGen/X86/dwarf_eh_resume.ll b/llvm/test/CodeGen/X86/dwarf_eh_resume.ll index 90fad00b68bd..d66d89d16a39 100644 --- a/llvm/test/CodeGen/X86/dwarf_eh_resume.ll +++ b/llvm/test/CodeGen/X86/dwarf_eh_resume.ll @@ -1,5 +1,5 @@ -; RUN: opt -mtriple=x86_64-linux-gnu -dwarfehprepare -S %s | FileCheck %s -; RUN: opt -mtriple=x86_64-linux-gnu -passes=dwarfehprepare -S %s | FileCheck %s +; RUN: opt -mtriple=x86_64-linux-gnu -dwarf-eh-prepare -S %s | FileCheck %s +; RUN: opt -mtriple=x86_64-linux-gnu -passes=dwarf-eh-prepare -S %s | FileCheck %s declare i32 @hoge(...) diff --git a/llvm/test/CodeGen/X86/evex-to-vex-compress.mir b/llvm/test/CodeGen/X86/evex-to-vex-compress.mir index 9c49ff8e573e..06d3c1532c3e 100644 --- a/llvm/test/CodeGen/X86/evex-to-vex-compress.mir +++ b/llvm/test/CodeGen/X86/evex-to-vex-compress.mir @@ -4,8 +4,8 @@ --- | define void @evex_z256_to_vex_test() { ret void } - define void @evex_z128_to_vex_test() { ret void } - define void @evex_scalar_to_vex_test() { ret void } + define void @evex_z128_to_vex_test() { ret void } + define void @evex_scalar_to_vex_test() { ret void } define void @evex_z256_to_evex_test() { ret void } define void @evex_z128_to_evex_test() { ret void } define void @evex_scalar_to_evex_test() { ret void } @@ -14,141 +14,141 @@ # CHECK-LABEL: name: evex_z256_to_vex_test # CHECK: bb.0: -name: evex_z256_to_vex_test -body: | - bb.0: +name: evex_z256_to_vex_test +body: | + bb.0: ; CHECK: VMOVAPDYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVAPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVAPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVAPDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVAPDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVAPDYrr $ymm0 - $ymm0 = VMOVAPDZ256rr $ymm0 + $ymm0 = VMOVAPDZ256rr $ymm0 ; CHECK: VMOVAPSYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVAPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVAPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVAPSYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVAPSZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVAPSYrr $ymm0 - $ymm0 = VMOVAPSZ256rr $ymm0 + $ymm0 = VMOVAPSZ256rr $ymm0 ; CHECK: $ymm0 = VMOVDDUPYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDDUPZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDDUPYrr $ymm0 - $ymm0 = VMOVDDUPZ256rr $ymm0 + $ymm0 = VMOVDDUPZ256rr $ymm0 ; CHECK: VMOVDQAYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVDQA32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVDQA32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVDQAYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDQA32Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDQAYrr $ymm0 - $ymm0 = VMOVDQA32Z256rr $ymm0 + $ymm0 = VMOVDQA32Z256rr $ymm0 ; CHECK: VMOVDQAYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVDQA64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVDQA64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVDQAYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDQA64Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDQAYrr $ymm0 - $ymm0 = VMOVDQA64Z256rr $ymm0 + $ymm0 = VMOVDQA64Z256rr $ymm0 ; CHECK: VMOVDQUYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVDQU16Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVDQU16Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVDQUYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDQU16Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDQUYrr $ymm0 - $ymm0 = VMOVDQU16Z256rr $ymm0 + $ymm0 = VMOVDQU16Z256rr $ymm0 ; CHECK: VMOVDQUYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVDQU32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVDQU32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVDQUYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDQU32Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDQUYrr $ymm0 - $ymm0 = VMOVDQU32Z256rr $ymm0 + $ymm0 = VMOVDQU32Z256rr $ymm0 ; CHECK: VMOVDQUYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVDQU64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVDQU64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVDQUYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDQU64Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDQUYrr $ymm0 - $ymm0 = VMOVDQU64Z256rr $ymm0 + $ymm0 = VMOVDQU64Z256rr $ymm0 ; CHECK: VMOVDQUYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVDQU8Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVDQU8Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVDQUYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVDQU8Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVDQUYrr $ymm0 - $ymm0 = VMOVDQU8Z256rr $ymm0 + $ymm0 = VMOVDQU8Z256rr $ymm0 ; CHECK: $ymm0 = VMOVNTDQAYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVNTDQAZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: VMOVNTDQYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVNTDQZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVNTDQZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: VMOVNTPDYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVNTPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVNTPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: VMOVNTPSYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 VMOVNTPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVSHDUPYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVSHDUPZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVSHDUPYrr $ymm0 - $ymm0 = VMOVSHDUPZ256rr $ymm0 + $ymm0 = VMOVSHDUPZ256rr $ymm0 ; CHECK: $ymm0 = VMOVSLDUPYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVSLDUPZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVSLDUPYrr $ymm0 - $ymm0 = VMOVSLDUPZ256rr $ymm0 + $ymm0 = VMOVSLDUPZ256rr $ymm0 ; CHECK: VMOVUPDYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVUPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVUPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VMOVUPDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VMOVUPDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVUPDYrr $ymm0 - $ymm0 = VMOVUPDZ256rr $ymm0 + $ymm0 = VMOVUPDZ256rr $ymm0 ; CHECK: VMOVUPSYmr $rdi, 1, $noreg, 0, $noreg, $ymm0 - VMOVUPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 + VMOVUPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm0 ; CHECK: $ymm0 = VPANDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPANDDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPANDYrr $ymm0, $ymm1 - $ymm0 = VPANDDZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPANDYrr $ymm0, $ymm1 + $ymm0 = VPANDDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPANDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPANDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPANDYrr $ymm0, $ymm1 - $ymm0 = VPANDQZ256rr $ymm0, $ymm1 + $ymm0 = VPANDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPANDNYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPANDNDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPANDNYrr $ymm0, $ymm1 - $ymm0 = VPANDNDZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPANDNYrr $ymm0, $ymm1 + $ymm0 = VPANDNDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPANDNYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPANDNQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPANDNYrr $ymm0, $ymm1 - $ymm0 = VPANDNQZ256rr $ymm0, $ymm1 + $ymm0 = VPANDNQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPAVGBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPAVGBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPAVGBYrr $ymm0, $ymm1 - $ymm0 = VPAVGBZ256rr $ymm0, $ymm1 + $ymm0 = VPAVGBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPAVGWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPAVGWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPAVGWYrr $ymm0, $ymm1 - $ymm0 = VPAVGWZ256rr $ymm0, $ymm1 + $ymm0 = VPAVGWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPADDBYrr $ymm0, $ymm1 - $ymm0 = VPADDBZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPADDBYrr $ymm0, $ymm1 + $ymm0 = VPADDBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDDYrr $ymm0, $ymm1 - $ymm0 = VPADDDZ256rr $ymm0, $ymm1 + $ymm0 = VPADDDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDQYrr $ymm0, $ymm1 - $ymm0 = VPADDQZ256rr $ymm0, $ymm1 + $ymm0 = VPADDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDSBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDSBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDSBYrr $ymm0, $ymm1 - $ymm0 = VPADDSBZ256rr $ymm0, $ymm1 + $ymm0 = VPADDSBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDSWYrr $ymm0, $ymm1 - $ymm0 = VPADDSWZ256rr $ymm0, $ymm1 + $ymm0 = VPADDSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDUSBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDUSBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDUSBYrr $ymm0, $ymm1 - $ymm0 = VPADDUSBZ256rr $ymm0, $ymm1 + $ymm0 = VPADDUSBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDUSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDUSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDUSWYrr $ymm0, $ymm1 - $ymm0 = VPADDUSWZ256rr $ymm0, $ymm1 + $ymm0 = VPADDUSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPADDWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPADDWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPADDWYrr $ymm0, $ymm1 - $ymm0 = VPADDWZ256rr $ymm0, $ymm1 + $ymm0 = VPADDWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VMULPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm0 = VMULPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm0 = VMULPDYrr $ymm0, $ymm1, implicit $mxcsr @@ -160,143 +160,143 @@ body: | ; CHECK: $ymm0 = VORPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VORPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VORPDYrr $ymm0, $ymm1 - $ymm0 = VORPDZ256rr $ymm0, $ymm1 + $ymm0 = VORPDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VORPSYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VORPSZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VORPSYrr $ymm0, $ymm1 - $ymm0 = VORPSZ256rr $ymm0, $ymm1 + $ymm0 = VORPSZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMADDUBSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMADDUBSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMADDUBSWYrr $ymm0, $ymm1 - $ymm0 = VPMADDUBSWZ256rr $ymm0, $ymm1 + $ymm0 = VPMADDUBSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMADDWDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMADDWDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMADDWDYrr $ymm0, $ymm1 - $ymm0 = VPMADDWDZ256rr $ymm0, $ymm1 + $ymm0 = VPMADDWDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMAXSBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMAXSBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMAXSBYrr $ymm0, $ymm1 - $ymm0 = VPMAXSBZ256rr $ymm0, $ymm1 + $ymm0 = VPMAXSBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMAXSDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMAXSDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMAXSDYrr $ymm0, $ymm1 - $ymm0 = VPMAXSDZ256rr $ymm0, $ymm1 + $ymm0 = VPMAXSDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMAXSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMAXSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMAXSWYrr $ymm0, $ymm1 - $ymm0 = VPMAXSWZ256rr $ymm0, $ymm1 + $ymm0 = VPMAXSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMAXUBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMAXUBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMAXUBYrr $ymm0, $ymm1 - $ymm0 = VPMAXUBZ256rr $ymm0, $ymm1 + $ymm0 = VPMAXUBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMAXUDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMAXUDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMAXUDYrr $ymm0, $ymm1 - $ymm0 = VPMAXUDZ256rr $ymm0, $ymm1 + $ymm0 = VPMAXUDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMAXUWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMAXUWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMAXUWYrr $ymm0, $ymm1 - $ymm0 = VPMAXUWZ256rr $ymm0, $ymm1 + $ymm0 = VPMAXUWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMINSBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMINSBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMINSBYrr $ymm0, $ymm1 - $ymm0 = VPMINSBZ256rr $ymm0, $ymm1 + $ymm0 = VPMINSBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMINSDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMINSDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMINSDYrr $ymm0, $ymm1 - $ymm0 = VPMINSDZ256rr $ymm0, $ymm1 + $ymm0 = VPMINSDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMINSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMINSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMINSWYrr $ymm0, $ymm1 - $ymm0 = VPMINSWZ256rr $ymm0, $ymm1 + $ymm0 = VPMINSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMINUBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMINUBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMINUBYrr $ymm0, $ymm1 - $ymm0 = VPMINUBZ256rr $ymm0, $ymm1 + $ymm0 = VPMINUBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMINUDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMINUDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMINUDYrr $ymm0, $ymm1 - $ymm0 = VPMINUDZ256rr $ymm0, $ymm1 + $ymm0 = VPMINUDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMINUWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMINUWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMINUWYrr $ymm0, $ymm1 - $ymm0 = VPMINUWZ256rr $ymm0, $ymm1 + $ymm0 = VPMINUWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPMULDQYrr $ymm0, $ymm1 - $ymm0 = VPMULDQZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPMULDQYrr $ymm0, $ymm1 + $ymm0 = VPMULDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULHRSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULHRSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMULHRSWYrr $ymm0, $ymm1 - $ymm0 = VPMULHRSWZ256rr $ymm0, $ymm1 + $ymm0 = VPMULHRSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULHUWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULHUWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMULHUWYrr $ymm0, $ymm1 - $ymm0 = VPMULHUWZ256rr $ymm0, $ymm1 + $ymm0 = VPMULHUWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULHWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULHWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMULHWYrr $ymm0, $ymm1 - $ymm0 = VPMULHWZ256rr $ymm0, $ymm1 + $ymm0 = VPMULHWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULLDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULLDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMULLDYrr $ymm0, $ymm1 - $ymm0 = VPMULLDZ256rr $ymm0, $ymm1 + $ymm0 = VPMULLDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULLWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULLWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPMULLWYrr $ymm0, $ymm1 - $ymm0 = VPMULLWZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPMULLWYrr $ymm0, $ymm1 + $ymm0 = VPMULLWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPMULUDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPMULUDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMULUDQYrr $ymm0, $ymm1 - $ymm0 = VPMULUDQZ256rr $ymm0, $ymm1 + $ymm0 = VPMULUDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPORYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPORDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPORYrr $ymm0, $ymm1 - $ymm0 = VPORDZ256rr $ymm0, $ymm1 + $ymm0 = VPORDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPORYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPORQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPORYrr $ymm0, $ymm1 - $ymm0 = VPORQZ256rr $ymm0, $ymm1 + $ymm0 = VPORQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBBYrr $ymm0, $ymm1 - $ymm0 = VPSUBBZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBDYrr $ymm0, $ymm1 - $ymm0 = VPSUBDZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBQYrr $ymm0, $ymm1 - $ymm0 = VPSUBQZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBSBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBSBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBSBYrr $ymm0, $ymm1 - $ymm0 = VPSUBSBZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBSBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBSWYrr $ymm0, $ymm1 - $ymm0 = VPSUBSWZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBUSBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBUSBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBUSBYrr $ymm0, $ymm1 - $ymm0 = VPSUBUSBZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBUSBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBUSWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBUSWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBUSWYrr $ymm0, $ymm1 - $ymm0 = VPSUBUSWZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBUSWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSUBWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSUBWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSUBWYrr $ymm0, $ymm1 - $ymm0 = VPSUBWZ256rr $ymm0, $ymm1 + $ymm0 = VPSUBWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPXORYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPXORDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPXORYrr $ymm0, $ymm1 - $ymm0 = VPXORDZ256rr $ymm0, $ymm1 + $ymm0 = VPXORDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPXORYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPXORQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPXORYrr $ymm0, $ymm1 - $ymm0 = VPXORQZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPXORYrr $ymm0, $ymm1 + $ymm0 = VPXORQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VADDPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm0 = VADDPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm0 = VADDPDYrr $ymm0, $ymm1, implicit $mxcsr @@ -308,19 +308,19 @@ body: | ; CHECK: $ymm0 = VANDNPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VANDNPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VANDNPDYrr $ymm0, $ymm1 - $ymm0 = VANDNPDZ256rr $ymm0, $ymm1 + $ymm0 = VANDNPDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VANDNPSYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VANDNPSZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VANDNPSYrr $ymm0, $ymm1 - $ymm0 = VANDNPSZ256rr $ymm0, $ymm1 + $ymm0 = VANDNPSZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VANDPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VANDPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VANDPDYrr $ymm0, $ymm1 - $ymm0 = VANDPDZ256rr $ymm0, $ymm1 + $ymm0 = VANDPDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VANDPSYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VANDPSZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VANDPSYrr $ymm0, $ymm1 - $ymm0 = VANDPSZ256rr $ymm0, $ymm1 + $ymm0 = VANDPSZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VDIVPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm0 = VDIVPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm0 = VDIVPDYrr $ymm0, $ymm1, implicit $mxcsr @@ -364,43 +364,43 @@ body: | ; CHECK: $ymm0 = VXORPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VXORPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VXORPDYrr $ymm0, $ymm1 - $ymm0 = VXORPDZ256rr $ymm0, $ymm1 + $ymm0 = VXORPDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VXORPSYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VXORPSZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VXORPSYrr $ymm0, $ymm1 - $ymm0 = VXORPSZ256rr $ymm0, $ymm1 + $ymm0 = VXORPSZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPACKSSDWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPACKSSDWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPACKSSDWYrr $ymm0, $ymm1 - $ymm0 = VPACKSSDWZ256rr $ymm0, $ymm1 + $ymm0 = VPACKSSDWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPACKSSWBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPACKSSWBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPACKSSWBYrr $ymm0, $ymm1 - $ymm0 = VPACKSSWBZ256rr $ymm0, $ymm1 + $ymm0 = VPACKSSWBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPACKUSDWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPACKUSDWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPACKUSDWYrr $ymm0, $ymm1 - $ymm0 = VPACKUSDWZ256rr $ymm0, $ymm1 + $ymm0 = VPACKUSDWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPACKUSWBYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPACKUSWBZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPACKUSWBYrr $ymm0, $ymm1 - $ymm0 = VPACKUSWBZ256rr $ymm0, $ymm1 + $ymm0 = VPACKUSWBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VUNPCKHPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VUNPCKHPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VUNPCKHPDYrr $ymm0, $ymm1 - $ymm0 = VUNPCKHPDZ256rr $ymm0, $ymm1 + $ymm0 = VUNPCKHPDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VUNPCKHPSYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VUNPCKHPSZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VUNPCKHPSYrr $ymm0, $ymm1 - $ymm0 = VUNPCKHPSZ256rr $ymm0, $ymm1 + $ymm0 = VUNPCKHPSZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VUNPCKLPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VUNPCKLPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VUNPCKLPDYrr $ymm0, $ymm1 - $ymm0 = VUNPCKLPDZ256rr $ymm0, $ymm1 + $ymm0 = VUNPCKLPDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VUNPCKLPSYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VUNPCKLPSZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VUNPCKLPSYrr $ymm0, $ymm1 - $ymm0 = VUNPCKLPSZ256rr $ymm0, $ymm1 + $ymm0 = VUNPCKLPSZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VSUBPDYrm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm0 = VSUBPDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm0 = VSUBPDYrr $ymm0, $ymm1, implicit $mxcsr @@ -412,35 +412,35 @@ body: | ; CHECK: $ymm0 = VPUNPCKHBWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKHBWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPUNPCKHBWYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKHBWZ256rr $ymm0, $ymm1 + $ymm0 = VPUNPCKHBWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKHDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKHDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPUNPCKHDQYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKHDQZ256rr $ymm0, $ymm1 + $ymm0 = VPUNPCKHDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKHQDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKHQDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPUNPCKHQDQYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKHQDQZ256rr $ymm0, $ymm1 + $ymm0 = VPUNPCKHQDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKHWDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKHWDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPUNPCKHWDYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKHWDZ256rr $ymm0, $ymm1 + $ymm0 = VPUNPCKHWDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKLBWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKLBWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPUNPCKLBWYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKLBWZ256rr $ymm0, $ymm1 + $ymm0 = VPUNPCKLBWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKLDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKLDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPUNPCKLDQYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKLDQZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPUNPCKLDQYrr $ymm0, $ymm1 + $ymm0 = VPUNPCKLDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKLQDQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKLQDQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPUNPCKLQDQYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKLQDQZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPUNPCKLQDQYrr $ymm0, $ymm1 + $ymm0 = VPUNPCKLQDQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPUNPCKLWDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPUNPCKLWDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPUNPCKLWDYrr $ymm0, $ymm1 - $ymm0 = VPUNPCKLWDZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPUNPCKLWDYrr $ymm0, $ymm1 + $ymm0 = VPUNPCKLWDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VFMADD132PDYm $ymm0, $ymm0, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm0 = VFMADD132PDZ256m $ymm0, $ymm0, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm0 = VFMADD132PDYr $ymm0, $ymm1, $ymm2, implicit $mxcsr @@ -586,98 +586,98 @@ body: | ; CHECK: $ymm0 = VFNMSUB231PSYr $ymm0, $ymm1, $ymm2, implicit $mxcsr $ymm0 = VFNMSUB231PSZ256r $ymm0, $ymm1, $ymm2, implicit $mxcsr ; CHECK: $ymm0 = VPSRADYri $ymm0, 7 - $ymm0 = VPSRADZ256ri $ymm0, 7 + $ymm0 = VPSRADZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSRADYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRADZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRADYrr $ymm0, $xmm1 - $ymm0 = VPSRADZ256rr $ymm0, $xmm1 + $ymm0 = VPSRADZ256rr $ymm0, $xmm1 ; CHECK: $ymm0 = VPSRAVDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRAVDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRAVDYrr $ymm0, $ymm1 - $ymm0 = VPSRAVDZ256rr $ymm0, $ymm1 + $ymm0 = VPSRAVDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSRAWYri $ymm0, 7 - $ymm0 = VPSRAWZ256ri $ymm0, 7 + $ymm0 = VPSRAWZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSRAWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRAWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRAWYrr $ymm0, $xmm1 - $ymm0 = VPSRAWZ256rr $ymm0, $xmm1 + $ymm0 = VPSRAWZ256rr $ymm0, $xmm1 ; CHECK: $ymm0 = VPSRLDQYri $ymm0, 7 $ymm0 = VPSRLDQZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSRLDYri $ymm0, 7 - $ymm0 = VPSRLDZ256ri $ymm0, 7 + $ymm0 = VPSRLDZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSRLDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRLDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRLDYrr $ymm0, $xmm1 - $ymm0 = VPSRLDZ256rr $ymm0, $xmm1 + $ymm0 = VPSRLDZ256rr $ymm0, $xmm1 ; CHECK: $ymm0 = VPSRLQYri $ymm0, 7 - $ymm0 = VPSRLQZ256ri $ymm0, 7 + $ymm0 = VPSRLQZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSRLQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRLQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRLQYrr $ymm0, $xmm1 - $ymm0 = VPSRLQZ256rr $ymm0, $xmm1 + $ymm0 = VPSRLQZ256rr $ymm0, $xmm1 ; CHECK: $ymm0 = VPSRLVDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRLVDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRLVDYrr $ymm0, $ymm1 - $ymm0 = VPSRLVDZ256rr $ymm0, $ymm1 + $ymm0 = VPSRLVDZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSRLVQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRLVQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSRLVQYrr $ymm0, $ymm1 - $ymm0 = VPSRLVQZ256rr $ymm0, $ymm1 + $ymm0 = VPSRLVQZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSRLWYri $ymm0, 7 - $ymm0 = VPSRLWZ256ri $ymm0, 7 + $ymm0 = VPSRLWZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSRLWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSRLWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPSRLWYrr $ymm0, $xmm1 - $ymm0 = VPSRLWZ256rr $ymm0, $xmm1 + ; CHECK: $ymm0 = VPSRLWYrr $ymm0, $xmm1 + $ymm0 = VPSRLWZ256rr $ymm0, $xmm1 ; CHECK: $ymm0 = VPMOVSXBDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVSXBDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVSXBDYrr $xmm0 - $ymm0 = VPMOVSXBDZ256rr $xmm0 + $ymm0 = VPMOVSXBDZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVSXBQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVSXBQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVSXBQYrr $xmm0 - $ymm0 = VPMOVSXBQZ256rr $xmm0 + $ymm0 = VPMOVSXBQZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVSXBWYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVSXBWZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVSXBWYrr $xmm0 - $ymm0 = VPMOVSXBWZ256rr $xmm0 + $ymm0 = VPMOVSXBWZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVSXDQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVSXDQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVSXDQYrr $xmm0 - $ymm0 = VPMOVSXDQZ256rr $xmm0 + $ymm0 = VPMOVSXDQZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVSXWDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVSXWDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVSXWDYrr $xmm0 - $ymm0 = VPMOVSXWDZ256rr $xmm0 + $ymm0 = VPMOVSXWDZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVSXWQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVSXWQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVSXWQYrr $xmm0 - $ymm0 = VPMOVSXWQZ256rr $xmm0 + $ymm0 = VPMOVSXWQZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVZXBDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVZXBDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVZXBDYrr $xmm0 - $ymm0 = VPMOVZXBDZ256rr $xmm0 + $ymm0 = VPMOVZXBDZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVZXBQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVZXBQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVZXBQYrr $xmm0 - $ymm0 = VPMOVZXBQZ256rr $xmm0 + $ymm0 = VPMOVZXBQZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVZXBWYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVZXBWZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVZXBWYrr $xmm0 - $ymm0 = VPMOVZXBWZ256rr $xmm0 + $ymm0 = VPMOVZXBWZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVZXDQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVZXDQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVZXDQYrr $xmm0 - $ymm0 = VPMOVZXDQZ256rr $xmm0 + $ymm0 = VPMOVZXDQZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVZXWDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVZXWDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPMOVZXWDYrr $xmm0 - $ymm0 = VPMOVZXWDZ256rr $xmm0 + $ymm0 = VPMOVZXWDZ256rr $xmm0 ; CHECK: $ymm0 = VPMOVZXWQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPMOVZXWQZ256rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPMOVZXWQYrr $xmm0 - $ymm0 = VPMOVZXWQZ256rr $xmm0 - ; CHECK: $ymm0 = VBROADCASTF128 $rip, 1, $noreg, 0, $noreg + ; CHECK: $ymm0 = VPMOVZXWQYrr $xmm0 + $ymm0 = VPMOVZXWQZ256rr $xmm0 + ; CHECK: $ymm0 = VBROADCASTF128rm $rip, 1, $noreg, 0, $noreg $ymm0 = VBROADCASTF32X4Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VBROADCASTSDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VBROADCASTF32X2Z256rm $rip, 1, $noreg, 0, $noreg @@ -686,24 +686,24 @@ body: | ; CHECK: $ymm0 = VBROADCASTSDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VBROADCASTSDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VBROADCASTSDYrr $xmm0 - $ymm0 = VBROADCASTSDZ256rr $xmm0 + $ymm0 = VBROADCASTSDZ256rr $xmm0 ; CHECK: $ymm0 = VBROADCASTSSYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VBROADCASTSSZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VBROADCASTSSYrr $xmm0 - $ymm0 = VBROADCASTSSZ256rr $xmm0 + $ymm0 = VBROADCASTSSZ256rr $xmm0 ; CHECK: $ymm0 = VPBROADCASTBYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPBROADCASTBZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPBROADCASTBYrr $xmm0 - $ymm0 = VPBROADCASTBZ256rr $xmm0 + $ymm0 = VPBROADCASTBZ256rr $xmm0 ; CHECK: $ymm0 = VPBROADCASTDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPBROADCASTDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPBROADCASTDYrr $xmm0 - $ymm0 = VPBROADCASTDZ256rr $xmm0 + $ymm0 = VPBROADCASTDZ256rr $xmm0 ; CHECK: $ymm0 = VPBROADCASTWYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPBROADCASTWZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPBROADCASTWYrr $xmm0 - $ymm0 = VPBROADCASTWZ256rr $xmm0 - ; CHECK: $ymm0 = VBROADCASTI128 $rip, 1, $noreg, 0, $noreg + $ymm0 = VPBROADCASTWZ256rr $xmm0 + ; CHECK: $ymm0 = VBROADCASTI128rm $rip, 1, $noreg, 0, $noreg $ymm0 = VBROADCASTI32X4Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPBROADCASTQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VBROADCASTI32X2Z256rm $rip, 1, $noreg, 0, $noreg @@ -711,66 +711,66 @@ body: | $ymm0 = VBROADCASTI32X2Z256rr $xmm0 ; CHECK: $ymm0 = VPBROADCASTQYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPBROADCASTQZ256rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPBROADCASTQYrr $xmm0 - $ymm0 = VPBROADCASTQZ256rr $xmm0 + ; CHECK: $ymm0 = VPBROADCASTQYrr $xmm0 + $ymm0 = VPBROADCASTQZ256rr $xmm0 ; CHECK: $ymm0 = VPABSBYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPABSBZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPABSBYrr $ymm0 - $ymm0 = VPABSBZ256rr $ymm0 + $ymm0 = VPABSBZ256rr $ymm0 ; CHECK: $ymm0 = VPABSDYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPABSDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPABSDYrr $ymm0 - $ymm0 = VPABSDZ256rr $ymm0 + $ymm0 = VPABSDZ256rr $ymm0 ; CHECK: $ymm0 = VPABSWYrm $rip, 1, $noreg, 0, $noreg $ymm0 = VPABSWZ256rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPABSWYrr $ymm0 - $ymm0 = VPABSWZ256rr $ymm0 + ; CHECK: $ymm0 = VPABSWYrr $ymm0 + $ymm0 = VPABSWZ256rr $ymm0 ; CHECK: $ymm0 = VPSADBWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSADBWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm0 = VPSADBWYrr $ymm0, $ymm1 - $ymm0 = VPSADBWZ256rr $ymm0, $ymm1 + ; CHECK: $ymm0 = VPSADBWYrr $ymm0, $ymm1 + $ymm0 = VPSADBWZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPERMDYrm $ymm0, $rdi, 1, $noreg, 0, $noreg - $ymm0 = VPERMDZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg + $ymm0 = VPERMDZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPERMDYrr $ymm1, $ymm0 - $ymm0 = VPERMDZ256rr $ymm1, $ymm0 + $ymm0 = VPERMDZ256rr $ymm1, $ymm0 ; CHECK: $ymm0 = VPERMILPDYmi $rdi, 1, $noreg, 0, $noreg, 7 $ymm0 = VPERMILPDZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm0 = VPERMILPDYri $ymm0, 7 - $ymm0 = VPERMILPDZ256ri $ymm0, 7 + $ymm0 = VPERMILPDZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPERMILPDYrm $ymm0, $rdi, 1, $noreg, 0, $noreg - $ymm0 = VPERMILPDZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg + $ymm0 = VPERMILPDZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPERMILPDYrr $ymm1, $ymm0 - $ymm0 = VPERMILPDZ256rr $ymm1, $ymm0 + $ymm0 = VPERMILPDZ256rr $ymm1, $ymm0 ; CHECK: $ymm0 = VPERMILPSYmi $rdi, 1, $noreg, 0, $noreg, 7 $ymm0 = VPERMILPSZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm0 = VPERMILPSYri $ymm0, 7 - $ymm0 = VPERMILPSZ256ri $ymm0, 7 + $ymm0 = VPERMILPSZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPERMILPSYrm $ymm0, $rdi, 1, $noreg, 0, $noreg - $ymm0 = VPERMILPSZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg + $ymm0 = VPERMILPSZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPERMILPSYrr $ymm1, $ymm0 - $ymm0 = VPERMILPSZ256rr $ymm1, $ymm0 + $ymm0 = VPERMILPSZ256rr $ymm1, $ymm0 ; CHECK: $ymm0 = VPERMPDYmi $rdi, 1, $noreg, 0, $noreg, 7 $ymm0 = VPERMPDZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm0 = VPERMPDYri $ymm0, 7 - $ymm0 = VPERMPDZ256ri $ymm0, 7 + $ymm0 = VPERMPDZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPERMPSYrm $ymm0, $rdi, 1, $noreg, 0, $noreg - $ymm0 = VPERMPSZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg + $ymm0 = VPERMPSZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPERMPSYrr $ymm1, $ymm0 - $ymm0 = VPERMPSZ256rr $ymm1, $ymm0 + $ymm0 = VPERMPSZ256rr $ymm1, $ymm0 ; CHECK: $ymm0 = VPERMQYmi $rdi, 1, $noreg, 0, $noreg, 7 $ymm0 = VPERMQZ256mi $rdi, 1, $noreg, 0, $noreg, 7 - ; CHECK: $ymm0 = VPERMQYri $ymm0, 7 - $ymm0 = VPERMQZ256ri $ymm0, 7 + ; CHECK: $ymm0 = VPERMQYri $ymm0, 7 + $ymm0 = VPERMQZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSLLDQYri $ymm0, 14 - $ymm0 = VPSLLDQZ256ri $ymm0, 14 + $ymm0 = VPSLLDQZ256ri $ymm0, 14 ; CHECK: $ymm0 = VPSLLDYri $ymm0, 7 - $ymm0 = VPSLLDZ256ri $ymm0, 7 + $ymm0 = VPSLLDZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSLLDYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSLLDZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSLLDYrr $ymm0, $xmm0 $ymm0 = VPSLLDZ256rr $ymm0, $xmm0 ; CHECK: $ymm0 = VPSLLQYri $ymm0, 7 - $ymm0 = VPSLLQZ256ri $ymm0, 7 + $ymm0 = VPSLLQZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSLLQYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSLLQZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSLLQYrr $ymm0, $xmm0 @@ -784,7 +784,7 @@ body: | ; CHECK: $ymm0 = VPSLLVQYrr $ymm0, $ymm0 $ymm0 = VPSLLVQZ256rr $ymm0, $ymm0 ; CHECK: $ymm0 = VPSLLWYri $ymm0, 7 - $ymm0 = VPSLLWZ256ri $ymm0, 7 + $ymm0 = VPSLLWZ256ri $ymm0, 7 ; CHECK: $ymm0 = VPSLLWYrm $ymm0, $rip, 1, $noreg, 0, $noreg $ymm0 = VPSLLWZ256rm $ymm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSLLWYrr $ymm0, $xmm0 @@ -792,7 +792,7 @@ body: | ; CHECK: $ymm0 = VCVTDQ2PDYrm $rdi, 1, $noreg, 0, $noreg $ymm0 = VCVTDQ2PDZ256rm $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VCVTDQ2PDYrr $xmm0 - $ymm0 = VCVTDQ2PDZ256rr $xmm0 + $ymm0 = VCVTDQ2PDZ256rr $xmm0 ; CHECK: $ymm0 = VCVTDQ2PSYrm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm0 = VCVTDQ2PSZ256rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm0 = VCVTDQ2PSYrr $ymm0, implicit $mxcsr @@ -841,26 +841,26 @@ body: | $ymm0 = VPALIGNRZ256rmi $ymm0, $rdi, 1, $noreg, 0, $noreg, 1 ; CHECK: $ymm0 = VPALIGNRYrri $ymm0, $ymm1, 1 $ymm0 = VPALIGNRZ256rri $ymm0, $ymm1, 1 - ; CHECK: $ymm0 = VMOVUPSYrm $rdi, 1, $noreg, 0, $noreg - $ymm0 = VMOVUPSZ256rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $ymm0 = VMOVUPSYrm $rdi, 1, $noreg, 0, $noreg + $ymm0 = VMOVUPSZ256rm $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VMOVUPSYrr $ymm0 - $ymm0 = VMOVUPSZ256rr $ymm0 + $ymm0 = VMOVUPSZ256rr $ymm0 ; CHECK: $ymm0 = VPSHUFBYrm $ymm0, $rdi, 1, $noreg, 0, $noreg $ymm0 = VPSHUFBZ256rm $ymm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm0 = VPSHUFBYrr $ymm0, $ymm1 - $ymm0 = VPSHUFBZ256rr $ymm0, $ymm1 + $ymm0 = VPSHUFBZ256rr $ymm0, $ymm1 ; CHECK: $ymm0 = VPSHUFDYmi $rdi, 1, $noreg, 0, $noreg, -24 $ymm0 = VPSHUFDZ256mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm0 = VPSHUFDYri $ymm0, -24 - $ymm0 = VPSHUFDZ256ri $ymm0, -24 + $ymm0 = VPSHUFDZ256ri $ymm0, -24 ; CHECK: $ymm0 = VPSHUFHWYmi $rdi, 1, $noreg, 0, $noreg, -24 $ymm0 = VPSHUFHWZ256mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm0 = VPSHUFHWYri $ymm0, -24 - $ymm0 = VPSHUFHWZ256ri $ymm0, -24 + $ymm0 = VPSHUFHWZ256ri $ymm0, -24 ; CHECK: $ymm0 = VPSHUFLWYmi $rdi, 1, $noreg, 0, $noreg, -24 $ymm0 = VPSHUFLWZ256mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm0 = VPSHUFLWYri $ymm0, -24 - $ymm0 = VPSHUFLWZ256ri $ymm0, -24 + $ymm0 = VPSHUFLWZ256ri $ymm0, -24 ; CHECK: $ymm0 = VSHUFPDYrmi $ymm0, $rdi, 1, $noreg, 0, $noreg, -24 $ymm0 = VSHUFPDZ256rmi $ymm0, $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm0 = VSHUFPDYrri $ymm0, $ymm1, -24 @@ -904,137 +904,137 @@ name: evex_z128_to_vex_test body: | bb.0: ; CHECK: VMOVAPDmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVAPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVAPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVAPDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVAPDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVAPDrr $xmm0 - $xmm0 = VMOVAPDZ128rr $xmm0 + $xmm0 = VMOVAPDZ128rr $xmm0 ; CHECK: VMOVAPSmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVAPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVAPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVAPSrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVAPSZ128rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VMOVAPSrr $xmm0 - $xmm0 = VMOVAPSZ128rr $xmm0 + ; CHECK: $xmm0 = VMOVAPSrr $xmm0 + $xmm0 = VMOVAPSZ128rr $xmm0 ; CHECK: VMOVDQAmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVDQA32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVDQA32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVDQArm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDQA32Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDQArr $xmm0 - $xmm0 = VMOVDQA32Z128rr $xmm0 + $xmm0 = VMOVDQA32Z128rr $xmm0 ; CHECK: VMOVDQAmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVDQA64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVDQA64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVDQArm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDQA64Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDQArr $xmm0 - $xmm0 = VMOVDQA64Z128rr $xmm0 + $xmm0 = VMOVDQA64Z128rr $xmm0 ; CHECK: VMOVDQUmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVDQU16Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVDQU16Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVDQUrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDQU16Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDQUrr $xmm0 - $xmm0 = VMOVDQU16Z128rr $xmm0 + $xmm0 = VMOVDQU16Z128rr $xmm0 ; CHECK: VMOVDQUmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVDQU32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVDQU32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVDQUrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDQU32Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDQUrr $xmm0 - $xmm0 = VMOVDQU32Z128rr $xmm0 + $xmm0 = VMOVDQU32Z128rr $xmm0 ; CHECK: VMOVDQUmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVDQU64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVDQU64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVDQUrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDQU64Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDQUrr $xmm0 - $xmm0 = VMOVDQU64Z128rr $xmm0 + $xmm0 = VMOVDQU64Z128rr $xmm0 ; CHECK: VMOVDQUmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVDQU8Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVDQU8Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVDQUrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDQU8Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDQUrr $xmm0 - $xmm0 = VMOVDQU8Z128rr $xmm0 + $xmm0 = VMOVDQU8Z128rr $xmm0 ; CHECK: $xmm0 = VMOVNTDQArm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVNTDQAZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: VMOVUPDmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVUPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVUPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVUPDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVUPDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVUPDrr $xmm0 - $xmm0 = VMOVUPDZ128rr $xmm0 + $xmm0 = VMOVUPDZ128rr $xmm0 ; CHECK: VMOVUPSmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVUPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVUPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVUPSrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVUPSZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVUPSrr $xmm0 - $xmm0 = VMOVUPSZ128rr $xmm0 + $xmm0 = VMOVUPSZ128rr $xmm0 ; CHECK: VMOVNTDQmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVNTDQZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVNTDQZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: VMOVNTPDmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVNTPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVNTPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: VMOVNTPSmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVNTPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVNTPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VPMOVSXBDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVSXBDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVSXBDrr $xmm0 - $xmm0 = VPMOVSXBDZ128rr $xmm0 + $xmm0 = VPMOVSXBDZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVSXBQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVSXBQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVSXBQrr $xmm0 - $xmm0 = VPMOVSXBQZ128rr $xmm0 + $xmm0 = VPMOVSXBQZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVSXBWrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVSXBWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVSXBWrr $xmm0 - $xmm0 = VPMOVSXBWZ128rr $xmm0 + $xmm0 = VPMOVSXBWZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVSXDQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVSXDQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVSXDQrr $xmm0 - $xmm0 = VPMOVSXDQZ128rr $xmm0 + $xmm0 = VPMOVSXDQZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVSXWDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVSXWDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVSXWDrr $xmm0 - $xmm0 = VPMOVSXWDZ128rr $xmm0 + $xmm0 = VPMOVSXWDZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVSXWQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVSXWQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVSXWQrr $xmm0 - $xmm0 = VPMOVSXWQZ128rr $xmm0 + $xmm0 = VPMOVSXWQZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVZXBDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVZXBDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVZXBDrr $xmm0 - $xmm0 = VPMOVZXBDZ128rr $xmm0 + $xmm0 = VPMOVZXBDZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVZXBQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVZXBQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVZXBQrr $xmm0 - $xmm0 = VPMOVZXBQZ128rr $xmm0 + $xmm0 = VPMOVZXBQZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVZXBWrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVZXBWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVZXBWrr $xmm0 - $xmm0 = VPMOVZXBWZ128rr $xmm0 + $xmm0 = VPMOVZXBWZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVZXDQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVZXDQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVZXDQrr $xmm0 - $xmm0 = VPMOVZXDQZ128rr $xmm0 + $xmm0 = VPMOVZXDQZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVZXWDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVZXWDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVZXWDrr $xmm0 - $xmm0 = VPMOVZXWDZ128rr $xmm0 + $xmm0 = VPMOVZXWDZ128rr $xmm0 ; CHECK: $xmm0 = VPMOVZXWQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPMOVZXWQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMOVZXWQrr $xmm0 - $xmm0 = VPMOVZXWQZ128rr $xmm0 + $xmm0 = VPMOVZXWQZ128rr $xmm0 ; CHECK: VMOVHPDmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVHPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVHPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVHPDrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVHPDZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVHPDZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: VMOVHPSmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVHPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVHPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVHPSrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVHPSZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVHPSZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: VMOVLPDmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVLPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + VMOVLPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVLPDrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVLPDZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVLPDZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: VMOVLPSmr $rdi, 1, $noreg, 0, $noreg, $xmm0 - VMOVLPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 - ; CHECK: $xmm0 = VMOVLPSrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVLPSZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg + VMOVLPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm0 + ; CHECK: $xmm0 = VMOVLPSrm $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVLPSZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMAXCPDrm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VMAXCPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VMAXCPDrr $xmm0, $xmm1, implicit $mxcsr @@ -1078,183 +1078,183 @@ body: | ; CHECK: $xmm0 = VORPDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VORPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VORPDrr $xmm0, $xmm1 - $xmm0 = VORPDZ128rr $xmm0, $xmm1 + $xmm0 = VORPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VORPSrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VORPSZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VORPSrr $xmm0, $xmm1 - $xmm0 = VORPSZ128rr $xmm0, $xmm1 + $xmm0 = VORPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDBrr $xmm0, $xmm1 - $xmm0 = VPADDBZ128rr $xmm0, $xmm1 + $xmm0 = VPADDBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDDrr $xmm0, $xmm1 - $xmm0 = VPADDDZ128rr $xmm0, $xmm1 + $xmm0 = VPADDDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDQrr $xmm0, $xmm1 - $xmm0 = VPADDQZ128rr $xmm0, $xmm1 + $xmm0 = VPADDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDSBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDSBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDSBrr $xmm0, $xmm1 - $xmm0 = VPADDSBZ128rr $xmm0, $xmm1 + $xmm0 = VPADDSBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDSWrr $xmm0, $xmm1 - $xmm0 = VPADDSWZ128rr $xmm0, $xmm1 + $xmm0 = VPADDSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDUSBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDUSBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDUSBrr $xmm0, $xmm1 - $xmm0 = VPADDUSBZ128rr $xmm0, $xmm1 + $xmm0 = VPADDUSBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDUSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDUSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDUSWrr $xmm0, $xmm1 - $xmm0 = VPADDUSWZ128rr $xmm0, $xmm1 + $xmm0 = VPADDUSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPADDWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPADDWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPADDWrr $xmm0, $xmm1 - $xmm0 = VPADDWZ128rr $xmm0, $xmm1 + $xmm0 = VPADDWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPANDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPANDDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPANDrr $xmm0, $xmm1 - $xmm0 = VPANDDZ128rr $xmm0, $xmm1 + $xmm0 = VPANDDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPANDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPANDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPANDrr $xmm0, $xmm1 - $xmm0 = VPANDQZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPANDrr $xmm0, $xmm1 + $xmm0 = VPANDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPANDNrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPANDNDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPANDNrr $xmm0, $xmm1 - $xmm0 = VPANDNDZ128rr $xmm0, $xmm1 + $xmm0 = VPANDNDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPANDNrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPANDNQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPANDNrr $xmm0, $xmm1 - $xmm0 = VPANDNQZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPANDNrr $xmm0, $xmm1 + $xmm0 = VPANDNQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPAVGBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPAVGBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPAVGBrr $xmm0, $xmm1 - $xmm0 = VPAVGBZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPAVGBrr $xmm0, $xmm1 + $xmm0 = VPAVGBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPAVGWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPAVGWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPAVGWrr $xmm0, $xmm1 - $xmm0 = VPAVGWZ128rr $xmm0, $xmm1 + $xmm0 = VPAVGWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMAXSBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMAXSBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMAXSBrr $xmm0, $xmm1 - $xmm0 = VPMAXSBZ128rr $xmm0, $xmm1 + $xmm0 = VPMAXSBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMAXSDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMAXSDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMAXSDrr $xmm0, $xmm1 - $xmm0 = VPMAXSDZ128rr $xmm0, $xmm1 + $xmm0 = VPMAXSDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMAXSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMAXSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPMAXSWrr $xmm0, $xmm1 - $xmm0 = VPMAXSWZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPMAXSWrr $xmm0, $xmm1 + $xmm0 = VPMAXSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMAXUBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMAXUBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMAXUBrr $xmm0, $xmm1 - $xmm0 = VPMAXUBZ128rr $xmm0, $xmm1 + $xmm0 = VPMAXUBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMAXUDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMAXUDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMAXUDrr $xmm0, $xmm1 - $xmm0 = VPMAXUDZ128rr $xmm0, $xmm1 + $xmm0 = VPMAXUDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMAXUWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMAXUWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMAXUWrr $xmm0, $xmm1 - $xmm0 = VPMAXUWZ128rr $xmm0, $xmm1 + $xmm0 = VPMAXUWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMINSBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMINSBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMINSBrr $xmm0, $xmm1 - $xmm0 = VPMINSBZ128rr $xmm0, $xmm1 + $xmm0 = VPMINSBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMINSDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMINSDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMINSDrr $xmm0, $xmm1 - $xmm0 = VPMINSDZ128rr $xmm0, $xmm1 + $xmm0 = VPMINSDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMINSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMINSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMINSWrr $xmm0, $xmm1 - $xmm0 = VPMINSWZ128rr $xmm0, $xmm1 + $xmm0 = VPMINSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMINUBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMINUBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMINUBrr $xmm0, $xmm1 - $xmm0 = VPMINUBZ128rr $xmm0, $xmm1 + $xmm0 = VPMINUBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMINUDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMINUDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMINUDrr $xmm0, $xmm1 - $xmm0 = VPMINUDZ128rr $xmm0, $xmm1 + $xmm0 = VPMINUDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMINUWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMINUWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMINUWrr $xmm0, $xmm1 - $xmm0 = VPMINUWZ128rr $xmm0, $xmm1 + $xmm0 = VPMINUWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULDQrr $xmm0, $xmm1 - $xmm0 = VPMULDQZ128rr $xmm0, $xmm1 + $xmm0 = VPMULDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULHRSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULHRSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULHRSWrr $xmm0, $xmm1 - $xmm0 = VPMULHRSWZ128rr $xmm0, $xmm1 + $xmm0 = VPMULHRSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULHUWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULHUWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULHUWrr $xmm0, $xmm1 - $xmm0 = VPMULHUWZ128rr $xmm0, $xmm1 + $xmm0 = VPMULHUWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULHWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULHWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULHWrr $xmm0, $xmm1 - $xmm0 = VPMULHWZ128rr $xmm0, $xmm1 + $xmm0 = VPMULHWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULLDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULLDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULLDrr $xmm0, $xmm1 - $xmm0 = VPMULLDZ128rr $xmm0, $xmm1 + $xmm0 = VPMULLDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULLWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULLWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULLWrr $xmm0, $xmm1 - $xmm0 = VPMULLWZ128rr $xmm0, $xmm1 + $xmm0 = VPMULLWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMULUDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMULUDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMULUDQrr $xmm0, $xmm1 - $xmm0 = VPMULUDQZ128rr $xmm0, $xmm1 + $xmm0 = VPMULUDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPORrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPORDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPORrr $xmm0, $xmm1 - $xmm0 = VPORDZ128rr $xmm0, $xmm1 + $xmm0 = VPORDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPORrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPORQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPORrr $xmm0, $xmm1 - $xmm0 = VPORQZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPORrr $xmm0, $xmm1 + $xmm0 = VPORQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSUBBrr $xmm0, $xmm1 - $xmm0 = VPSUBBZ128rr $xmm0, $xmm1 + $xmm0 = VPSUBBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSUBDrr $xmm0, $xmm1 - $xmm0 = VPSUBDZ128rr $xmm0, $xmm1 + $xmm0 = VPSUBDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSUBQrr $xmm0, $xmm1 - $xmm0 = VPSUBQZ128rr $xmm0, $xmm1 + $xmm0 = VPSUBQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBSBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBSBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPSUBSBrr $xmm0, $xmm1 - $xmm0 = VPSUBSBZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPSUBSBrr $xmm0, $xmm1 + $xmm0 = VPSUBSBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSUBSWrr $xmm0, $xmm1 - $xmm0 = VPSUBSWZ128rr $xmm0, $xmm1 + $xmm0 = VPSUBSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBUSBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBUSBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPSUBUSBrr $xmm0, $xmm1 - $xmm0 = VPSUBUSBZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPSUBUSBrr $xmm0, $xmm1 + $xmm0 = VPSUBUSBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBUSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBUSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSUBUSWrr $xmm0, $xmm1 - $xmm0 = VPSUBUSWZ128rr $xmm0, $xmm1 + $xmm0 = VPSUBUSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSUBWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSUBWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPSUBWrr $xmm0, $xmm1 - $xmm0 = VPSUBWZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPSUBWrr $xmm0, $xmm1 + $xmm0 = VPSUBWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VADDPDrm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VADDPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VADDPDrr $xmm0, $xmm1, implicit $mxcsr @@ -1266,19 +1266,19 @@ body: | ; CHECK: $xmm0 = VANDNPDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VANDNPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VANDNPDrr $xmm0, $xmm1 - $xmm0 = VANDNPDZ128rr $xmm0, $xmm1 + $xmm0 = VANDNPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VANDNPSrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VANDNPSZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VANDNPSrr $xmm0, $xmm1 - $xmm0 = VANDNPSZ128rr $xmm0, $xmm1 + $xmm0 = VANDNPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VANDPDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VANDPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VANDPDrr $xmm0, $xmm1 - $xmm0 = VANDPDZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VANDPDrr $xmm0, $xmm1 + $xmm0 = VANDPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VANDPSrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VANDPSZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VANDPSrr $xmm0, $xmm1 - $xmm0 = VANDPSZ128rr $xmm0, $xmm1 + $xmm0 = VANDPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VDIVPDrm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VDIVPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VDIVPDrr $xmm0, $xmm1, implicit $mxcsr @@ -1290,11 +1290,11 @@ body: | ; CHECK: $xmm0 = VPXORrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPXORDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPXORrr $xmm0, $xmm1 - $xmm0 = VPXORDZ128rr $xmm0, $xmm1 + $xmm0 = VPXORDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPXORrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPXORQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPXORrr $xmm0, $xmm1 - $xmm0 = VPXORQZ128rr $xmm0, $xmm1 + $xmm0 = VPXORQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VSUBPDrm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VSUBPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VSUBPDrr $xmm0, $xmm1, implicit $mxcsr @@ -1306,83 +1306,83 @@ body: | ; CHECK: $xmm0 = VXORPDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VXORPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VXORPDrr $xmm0, $xmm1 - $xmm0 = VXORPDZ128rr $xmm0, $xmm1 + $xmm0 = VXORPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VXORPSrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VXORPSZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VXORPSrr $xmm0, $xmm1 - $xmm0 = VXORPSZ128rr $xmm0, $xmm1 + $xmm0 = VXORPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMADDUBSWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMADDUBSWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPMADDUBSWrr $xmm0, $xmm1 - $xmm0 = VPMADDUBSWZ128rr $xmm0, $xmm1 + $xmm0 = VPMADDUBSWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPMADDWDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPMADDWDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPMADDWDrr $xmm0, $xmm1 - $xmm0 = VPMADDWDZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPMADDWDrr $xmm0, $xmm1 + $xmm0 = VPMADDWDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPACKSSDWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPACKSSDWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPACKSSDWrr $xmm0, $xmm1 - $xmm0 = VPACKSSDWZ128rr $xmm0, $xmm1 + $xmm0 = VPACKSSDWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPACKSSWBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPACKSSWBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPACKSSWBrr $xmm0, $xmm1 - $xmm0 = VPACKSSWBZ128rr $xmm0, $xmm1 + $xmm0 = VPACKSSWBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPACKUSDWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPACKUSDWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPACKUSDWrr $xmm0, $xmm1 - $xmm0 = VPACKUSDWZ128rr $xmm0, $xmm1 + $xmm0 = VPACKUSDWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPACKUSWBrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPACKUSWBZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPACKUSWBrr $xmm0, $xmm1 - $xmm0 = VPACKUSWBZ128rr $xmm0, $xmm1 + $xmm0 = VPACKUSWBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKHBWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKHBWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKHBWrr $xmm0, $xmm1 - $xmm0 = VPUNPCKHBWZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKHBWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKHDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKHDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKHDQrr $xmm0, $xmm1 - $xmm0 = VPUNPCKHDQZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKHDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKHQDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKHQDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKHQDQrr $xmm0, $xmm1 - $xmm0 = VPUNPCKHQDQZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKHQDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKHWDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKHWDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKHWDrr $xmm0, $xmm1 - $xmm0 = VPUNPCKHWDZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKHWDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKLBWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKLBWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKLBWrr $xmm0, $xmm1 - $xmm0 = VPUNPCKLBWZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKLBWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKLDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKLDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKLDQrr $xmm0, $xmm1 - $xmm0 = VPUNPCKLDQZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKLDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKLQDQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKLQDQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKLQDQrr $xmm0, $xmm1 - $xmm0 = VPUNPCKLQDQZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKLQDQZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPUNPCKLWDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPUNPCKLWDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPUNPCKLWDrr $xmm0, $xmm1 - $xmm0 = VPUNPCKLWDZ128rr $xmm0, $xmm1 + $xmm0 = VPUNPCKLWDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VUNPCKHPDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VUNPCKHPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VUNPCKHPDrr $xmm0, $xmm1 - $xmm0 = VUNPCKHPDZ128rr $xmm0, $xmm1 + $xmm0 = VUNPCKHPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VUNPCKHPSrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VUNPCKHPSZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VUNPCKHPSrr $xmm0, $xmm1 - $xmm0 = VUNPCKHPSZ128rr $xmm0, $xmm1 + $xmm0 = VUNPCKHPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VUNPCKLPDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VUNPCKLPDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VUNPCKLPDrr $xmm0, $xmm1 - $xmm0 = VUNPCKLPDZ128rr $xmm0, $xmm1 + $xmm0 = VUNPCKLPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VUNPCKLPSrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VUNPCKLPSZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VUNPCKLPSrr $xmm0, $xmm1 - $xmm0 = VUNPCKLPSZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VUNPCKLPSrr $xmm0, $xmm1 + $xmm0 = VUNPCKLPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VFMADD132PDm $xmm0, $xmm0, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VFMADD132PDZ128m $xmm0, $xmm0, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VFMADD132PDr $xmm0, $xmm1, $xmm2, implicit $mxcsr @@ -1528,13 +1528,13 @@ body: | ; CHECK: $xmm0 = VFNMSUB231PSr $xmm0, $xmm1, $xmm2, implicit $mxcsr $xmm0 = VFNMSUB231PSZ128r $xmm0, $xmm1, $xmm2, implicit $mxcsr ; CHECK: $xmm0 = VPSLLDri $xmm0, 7 - $xmm0 = VPSLLDZ128ri $xmm0, 7 + $xmm0 = VPSLLDZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSLLDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSLLDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSLLDrr $xmm0, $xmm0 $xmm0 = VPSLLDZ128rr $xmm0, $xmm0 ; CHECK: $xmm0 = VPSLLQri $xmm0, 7 - $xmm0 = VPSLLQZ128ri $xmm0, 7 + $xmm0 = VPSLLQZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSLLQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSLLQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSLLQrr $xmm0, $xmm0 @@ -1548,13 +1548,13 @@ body: | ; CHECK: $xmm0 = VPSLLVQrr $xmm0, $xmm0 $xmm0 = VPSLLVQZ128rr $xmm0, $xmm0 ; CHECK: $xmm0 = VPSLLWri $xmm0, 7 - $xmm0 = VPSLLWZ128ri $xmm0, 7 + $xmm0 = VPSLLWZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSLLWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSLLWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSLLWrr $xmm0, $xmm0 $xmm0 = VPSLLWZ128rr $xmm0, $xmm0 ; CHECK: $xmm0 = VPSRADri $xmm0, 7 - $xmm0 = VPSRADZ128ri $xmm0, 7 + $xmm0 = VPSRADZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSRADrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSRADZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSRADrr $xmm0, $xmm0 @@ -1563,22 +1563,22 @@ body: | $xmm0 = VPSRAVDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSRAVDrr $xmm0, $xmm0 $xmm0 = VPSRAVDZ128rr $xmm0, $xmm0 - ; CHECK: $xmm0 = VPSRAWri $xmm0, 7 - $xmm0 = VPSRAWZ128ri $xmm0, 7 + ; CHECK: $xmm0 = VPSRAWri $xmm0, 7 + $xmm0 = VPSRAWZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSRAWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSRAWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSRAWrr $xmm0, $xmm0 $xmm0 = VPSRAWZ128rr $xmm0, $xmm0 ; CHECK: $xmm0 = VPSRLDQri $xmm0, 14 - $xmm0 = VPSRLDQZ128ri $xmm0, 14 - ; CHECK: $xmm0 = VPSRLDri $xmm0, 7 - $xmm0 = VPSRLDZ128ri $xmm0, 7 + $xmm0 = VPSRLDQZ128ri $xmm0, 14 + ; CHECK: $xmm0 = VPSRLDri $xmm0, 7 + $xmm0 = VPSRLDZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSRLDrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSRLDZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSRLDrr $xmm0, $xmm0 $xmm0 = VPSRLDZ128rr $xmm0, $xmm0 - ; CHECK: $xmm0 = VPSRLQri $xmm0, 7 - $xmm0 = VPSRLQZ128ri $xmm0, 7 + ; CHECK: $xmm0 = VPSRLQri $xmm0, 7 + $xmm0 = VPSRLQZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSRLQrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSRLQZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSRLQrr $xmm0, $xmm0 @@ -1592,7 +1592,7 @@ body: | ; CHECK: $xmm0 = VPSRLVQrr $xmm0, $xmm0 $xmm0 = VPSRLVQZ128rr $xmm0, $xmm0 ; CHECK: $xmm0 = VPSRLWri $xmm0, 7 - $xmm0 = VPSRLWZ128ri $xmm0, 7 + $xmm0 = VPSRLWZ128ri $xmm0, 7 ; CHECK: $xmm0 = VPSRLWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSRLWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSRLWrr $xmm0, $xmm0 @@ -1600,27 +1600,27 @@ body: | ; CHECK: $xmm0 = VPERMILPDmi $rdi, 1, $noreg, 0, $noreg, 9 $xmm0 = VPERMILPDZ128mi $rdi, 1, $noreg, 0, $noreg, 9 ; CHECK: $xmm0 = VPERMILPDri $xmm0, 9 - $xmm0 = VPERMILPDZ128ri $xmm0, 9 + $xmm0 = VPERMILPDZ128ri $xmm0, 9 ; CHECK: $xmm0 = VPERMILPDrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VPERMILPDZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VPERMILPDZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPERMILPDrr $xmm0, $xmm1 - $xmm0 = VPERMILPDZ128rr $xmm0, $xmm1 + $xmm0 = VPERMILPDZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPERMILPSmi $rdi, 1, $noreg, 0, $noreg, 9 $xmm0 = VPERMILPSZ128mi $rdi, 1, $noreg, 0, $noreg, 9 ; CHECK: $xmm0 = VPERMILPSri $xmm0, 9 - $xmm0 = VPERMILPSZ128ri $xmm0, 9 + $xmm0 = VPERMILPSZ128ri $xmm0, 9 ; CHECK: $xmm0 = VPERMILPSrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VPERMILPSZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPERMILPSrr $xmm0, $xmm1 - $xmm0 = VPERMILPSZ128rr $xmm0, $xmm1 + $xmm0 = VPERMILPSZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm0 = VPERMILPSrr $xmm0, $xmm1 + $xmm0 = VPERMILPSZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VCVTPH2PSrm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VCVTPH2PSZ128rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VCVTPH2PSrr $xmm0, implicit $mxcsr $xmm0 = VCVTPH2PSZ128rr $xmm0, implicit $mxcsr ; CHECK: $xmm0 = VCVTDQ2PDrm $rdi, 1, $noreg, 0, $noreg $xmm0 = VCVTDQ2PDZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VCVTDQ2PDrr $xmm0 - $xmm0 = VCVTDQ2PDZ128rr $xmm0 + ; CHECK: $xmm0 = VCVTDQ2PDrr $xmm0 + $xmm0 = VCVTDQ2PDZ128rr $xmm0 ; CHECK: $xmm0 = VCVTDQ2PSrm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VCVTDQ2PSZ128rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VCVTDQ2PSrr $xmm0, implicit $mxcsr @@ -1657,34 +1657,34 @@ body: | $xmm0 = VSQRTPSZ128m $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VSQRTPSr $xmm0, implicit $mxcsr $xmm0 = VSQRTPSZ128r $xmm0, implicit $mxcsr - ; CHECK: $xmm0 = VMOVDDUPrm $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVDDUPZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VMOVDDUPrr $xmm0 - $xmm0 = VMOVDDUPZ128rr $xmm0 - ; CHECK: $xmm0 = VMOVSHDUPrm $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVSHDUPZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VMOVSHDUPrr $xmm0 - $xmm0 = VMOVSHDUPZ128rr $xmm0 - ; CHECK: $xmm0 = VMOVSLDUPrm $rdi, 1, $noreg, 0, $noreg - $xmm0 = VMOVSLDUPZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VMOVSLDUPrr $xmm0 - $xmm0 = VMOVSLDUPZ128rr $xmm0 + ; CHECK: $xmm0 = VMOVDDUPrm $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVDDUPZ128rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm0 = VMOVDDUPrr $xmm0 + $xmm0 = VMOVDDUPZ128rr $xmm0 + ; CHECK: $xmm0 = VMOVSHDUPrm $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVSHDUPZ128rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm0 = VMOVSHDUPrr $xmm0 + $xmm0 = VMOVSHDUPZ128rr $xmm0 + ; CHECK: $xmm0 = VMOVSLDUPrm $rdi, 1, $noreg, 0, $noreg + $xmm0 = VMOVSLDUPZ128rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm0 = VMOVSLDUPrr $xmm0 + $xmm0 = VMOVSLDUPZ128rr $xmm0 ; CHECK: $xmm0 = VPSHUFBrm $xmm0, $rdi, 1, $noreg, 0, $noreg $xmm0 = VPSHUFBZ128rm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPSHUFBrr $xmm0, $xmm1 - $xmm0 = VPSHUFBZ128rr $xmm0, $xmm1 + $xmm0 = VPSHUFBZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VPSHUFDmi $rdi, 1, $noreg, 0, $noreg, -24 $xmm0 = VPSHUFDZ128mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $xmm0 = VPSHUFDri $xmm0, -24 - $xmm0 = VPSHUFDZ128ri $xmm0, -24 + $xmm0 = VPSHUFDZ128ri $xmm0, -24 ; CHECK: $xmm0 = VPSHUFHWmi $rdi, 1, $noreg, 0, $noreg, -24 $xmm0 = VPSHUFHWZ128mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $xmm0 = VPSHUFHWri $xmm0, -24 - $xmm0 = VPSHUFHWZ128ri $xmm0, -24 + $xmm0 = VPSHUFHWZ128ri $xmm0, -24 ; CHECK: $xmm0 = VPSHUFLWmi $rdi, 1, $noreg, 0, $noreg, -24 $xmm0 = VPSHUFLWZ128mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $xmm0 = VPSHUFLWri $xmm0, -24 - $xmm0 = VPSHUFLWZ128ri $xmm0, -24 + $xmm0 = VPSHUFLWZ128ri $xmm0, -24 ; CHECK: $xmm0 = VPSLLDQri $xmm0, 7 $xmm0 = VPSLLDQZ128ri $xmm0, 7 ; CHECK: $xmm0 = VSHUFPDrmi $xmm0, $rip, 1, $noreg, 0, $noreg, -24 @@ -1697,28 +1697,28 @@ body: | $xmm0 = VSHUFPSZ128rri $xmm0, $xmm1, -24 ; CHECK: $xmm0 = VPSADBWrm $xmm0, $rip, 1, $noreg, 0, $noreg $xmm0 = VPSADBWZ128rm $xmm0, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPSADBWrr $xmm0, $xmm1 - $xmm0 = VPSADBWZ128rr $xmm0, $xmm1 + ; CHECK: $xmm0 = VPSADBWrr $xmm0, $xmm1 + $xmm0 = VPSADBWZ128rr $xmm0, $xmm1 ; CHECK: $xmm0 = VBROADCASTSSrm $rip, 1, $noreg, 0, $noreg $xmm0 = VBROADCASTSSZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VBROADCASTSSrr $xmm0 - $xmm0 = VBROADCASTSSZ128rr $xmm0 + $xmm0 = VBROADCASTSSZ128rr $xmm0 ; CHECK: $xmm0 = VPBROADCASTBrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPBROADCASTBZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPBROADCASTBrr $xmm0 - $xmm0 = VPBROADCASTBZ128rr $xmm0 + $xmm0 = VPBROADCASTBZ128rr $xmm0 ; CHECK: $xmm0 = VPBROADCASTDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPBROADCASTDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPBROADCASTDrr $xmm0 - $xmm0 = VPBROADCASTDZ128rr $xmm0 + $xmm0 = VPBROADCASTDZ128rr $xmm0 ; CHECK: $xmm0 = VPBROADCASTQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPBROADCASTQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPBROADCASTQrr $xmm0 - $xmm0 = VPBROADCASTQZ128rr $xmm0 + $xmm0 = VPBROADCASTQZ128rr $xmm0 ; CHECK: $xmm0 = VPBROADCASTWrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPBROADCASTWZ128rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VPBROADCASTWrr $xmm0 - $xmm0 = VPBROADCASTWZ128rr $xmm0 + ; CHECK: $xmm0 = VPBROADCASTWrr $xmm0 + $xmm0 = VPBROADCASTWZ128rr $xmm0 ; CHECK: $xmm0 = VPBROADCASTQrm $rip, 1, $noreg, 0, $noreg $xmm0 = VBROADCASTI32X2Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPBROADCASTQrr $xmm0 @@ -1730,15 +1730,15 @@ body: | ; CHECK: $xmm0 = VPABSBrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPABSBZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPABSBrr $xmm0 - $xmm0 = VPABSBZ128rr $xmm0 + $xmm0 = VPABSBZ128rr $xmm0 ; CHECK: $xmm0 = VPABSDrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPABSDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPABSDrr $xmm0 - $xmm0 = VPABSDZ128rr $xmm0 + $xmm0 = VPABSDZ128rr $xmm0 ; CHECK: $xmm0 = VPABSWrm $rip, 1, $noreg, 0, $noreg $xmm0 = VPABSWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VPABSWrr $xmm0 - $xmm0 = VPABSWZ128rr $xmm0 + $xmm0 = VPABSWZ128rr $xmm0 ; CHECK: $xmm0 = VPALIGNRrmi $xmm0, $rip, 1, $noreg, 0, $noreg, 15 $xmm0 = VPALIGNRZ128rmi $xmm0, $rip, 1, $noreg, 0, $noreg, 15 ; CHECK: $xmm0 = VPALIGNRrri $xmm0, $xmm1, 15 @@ -2074,38 +2074,38 @@ body: | $xmm0 = VFNMSUB231SSZr $xmm0, $xmm1, $xmm2, implicit $mxcsr ; CHECK: $xmm0 = VFNMSUB231SSr_Int $xmm0, $xmm1, $xmm2, implicit $mxcsr $xmm0 = VFNMSUB231SSZr_Int $xmm0, $xmm1, $xmm2, implicit $mxcsr - ; CHECK: VPEXTRBmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - VPEXTRBZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - ; CHECK: $eax = VPEXTRBrr $xmm0, 1 - $eax = VPEXTRBZrr $xmm0, 1 - ; CHECK: VPEXTRDmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - VPEXTRDZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - ; CHECK: $eax = VPEXTRDrr $xmm0, 1 - $eax = VPEXTRDZrr $xmm0, 1 - ; CHECK: VPEXTRQmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - VPEXTRQZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - ; CHECK: $rax = VPEXTRQrr $xmm0, 1 - $rax = VPEXTRQZrr $xmm0, 1 - ; CHECK: VPEXTRWmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - VPEXTRWZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 - ; CHECK: $eax = VPEXTRWrr $xmm0, 1 - $eax = VPEXTRWZrr $xmm0, 1 - ; CHECK: $xmm0 = VPINSRBrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm0 = VPINSRBZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm0 = VPINSRBrr $xmm0, $edi, 5 - $xmm0 = VPINSRBZrr $xmm0, $edi, 5 - ; CHECK: $xmm0 = VPINSRDrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm0 = VPINSRDZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm0 = VPINSRDrr $xmm0, $edi, 5 - $xmm0 = VPINSRDZrr $xmm0, $edi, 5 - ; CHECK: $xmm0 = VPINSRQrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm0 = VPINSRQZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm0 = VPINSRQrr $xmm0, $rdi, 5 - $xmm0 = VPINSRQZrr $xmm0, $rdi, 5 - ; CHECK: $xmm0 = VPINSRWrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm0 = VPINSRWZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm0 = VPINSRWrr $xmm0, $edi, 5 - $xmm0 = VPINSRWZrr $xmm0, $edi, 5 + ; CHECK: VPEXTRBmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + VPEXTRBZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + ; CHECK: $eax = VPEXTRBrr $xmm0, 1 + $eax = VPEXTRBZrr $xmm0, 1 + ; CHECK: VPEXTRDmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + VPEXTRDZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + ; CHECK: $eax = VPEXTRDrr $xmm0, 1 + $eax = VPEXTRDZrr $xmm0, 1 + ; CHECK: VPEXTRQmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + VPEXTRQZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + ; CHECK: $rax = VPEXTRQrr $xmm0, 1 + $rax = VPEXTRQZrr $xmm0, 1 + ; CHECK: VPEXTRWmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + VPEXTRWZmr $rdi, 1, $noreg, 0, $noreg, $xmm0, 3 + ; CHECK: $eax = VPEXTRWrr $xmm0, 1 + $eax = VPEXTRWZrr $xmm0, 1 + ; CHECK: $xmm0 = VPINSRBrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm0 = VPINSRBZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm0 = VPINSRBrr $xmm0, $edi, 5 + $xmm0 = VPINSRBZrr $xmm0, $edi, 5 + ; CHECK: $xmm0 = VPINSRDrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm0 = VPINSRDZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm0 = VPINSRDrr $xmm0, $edi, 5 + $xmm0 = VPINSRDZrr $xmm0, $edi, 5 + ; CHECK: $xmm0 = VPINSRQrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm0 = VPINSRQZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm0 = VPINSRQrr $xmm0, $rdi, 5 + $xmm0 = VPINSRQZrr $xmm0, $rdi, 5 + ; CHECK: $xmm0 = VPINSRWrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm0 = VPINSRWZrm $xmm0, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm0 = VPINSRWrr $xmm0, $edi, 5 + $xmm0 = VPINSRWZrr $xmm0, $edi, 5 ; CHECK: $xmm0 = VSQRTSDm $xmm0, $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm0 = VSQRTSDZm $xmm0, $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm0 = VSQRTSDm_Int $xmm0, $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr @@ -2135,9 +2135,9 @@ body: | ; CHECK: $xmm0 = VCVTSD2SSrr_Int $xmm0, $xmm1, implicit $mxcsr $xmm0 = VCVTSD2SSZrr_Int $xmm0, $xmm1, implicit $mxcsr ; CHECK: $xmm0 = VCVTSI2SDrm $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VCVTSI2SDZrm $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VCVTSI2SDZrm $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VCVTSI2SDrm_Int $xmm0, $rdi, 1, $noreg, 0, $noreg - $xmm0 = VCVTSI2SDZrm_Int $xmm0, $rdi, 1, $noreg, 0, $noreg + $xmm0 = VCVTSI2SDZrm_Int $xmm0, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VCVTSI2SDrr $xmm0, $edi $xmm0 = VCVTSI2SDZrr $xmm0, $edi ; CHECK: $xmm0 = VCVTSI2SDrr_Int $xmm0, $edi @@ -2214,10 +2214,10 @@ body: | $edi = VCVTTSS2SIZrr $xmm0, implicit $mxcsr ; CHECK: $edi = VCVTTSS2SIrr_Int $xmm0, implicit $mxcsr $edi = VCVTTSS2SIZrr_Int $xmm0, implicit $mxcsr - ; CHECK: $xmm0 = VMOV64toSDrr $rdi - $xmm0 = VMOV64toSDZrr $rdi + ; CHECK: $xmm0 = VMOV64toSDrr $rdi + $xmm0 = VMOV64toSDZrr $rdi ; CHECK: $xmm0 = VMOVDI2SSrr $eax - $xmm0 = VMOVDI2SSZrr $eax + $xmm0 = VMOVDI2SSZrr $eax ; CHECK: VMOVSDmr $rdi, 1, $noreg, 0, $noreg, $xmm0 VMOVSDZmr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVSDrm $rip, 1, $noreg, 0, $noreg @@ -2239,15 +2239,15 @@ body: | ; CHECK: $eax = VMOVSS2DIrr $xmm0 $eax = VMOVSS2DIZrr $xmm0 ; CHECK: $xmm0 = VMOV64toPQIrr $rdi - $xmm0 = VMOV64toPQIZrr $rdi + $xmm0 = VMOV64toPQIZrr $rdi ; CHECK: $xmm0 = VMOV64toPQIrm $rdi, 1, $noreg, 0, $noreg $xmm0 = VMOV64toPQIZrm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VMOV64toSDrr $rdi - $xmm0 = VMOV64toSDZrr $rdi + ; CHECK: $xmm0 = VMOV64toSDrr $rdi + $xmm0 = VMOV64toSDZrr $rdi ; CHECK: $xmm0 = VMOVDI2PDIrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVDI2PDIZrm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm0 = VMOVDI2PDIrr $edi - $xmm0 = VMOVDI2PDIZrr $edi + $xmm0 = VMOVDI2PDIZrr $edi ; CHECK: $xmm0 = VMOVLHPSrr $xmm0, $xmm1 $xmm0 = VMOVLHPSZrr $xmm0, $xmm1 ; CHECK: $xmm0 = VMOVHLPSrr $xmm0, $xmm1 @@ -2261,13 +2261,13 @@ body: | ; CHECK: VMOVPQI2QImr $rdi, 1, $noreg, 0, $noreg, $xmm0 VMOVPQI2QIZmr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $rdi = VMOVPQIto64rr $xmm0 - $rdi = VMOVPQIto64Zrr $xmm0 + $rdi = VMOVPQIto64Zrr $xmm0 ; CHECK: VMOVPQIto64mr $rdi, 1, $noreg, 0, $noreg, $xmm0 VMOVPQIto64Zmr $rdi, 1, $noreg, 0, $noreg, $xmm0 ; CHECK: $xmm0 = VMOVQI2PQIrm $rip, 1, $noreg, 0, $noreg $xmm0 = VMOVQI2PQIZrm $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm0 = VMOVZPQILo2PQIrr $xmm0 - $xmm0 = VMOVZPQILo2PQIZrr $xmm0 + ; CHECK: $xmm0 = VMOVZPQILo2PQIrr $xmm0 + $xmm0 = VMOVZPQILo2PQIZrr $xmm0 ; CHECK: VCOMISDrm_Int $xmm0, $rdi, 1, $noreg, 0, $noreg, implicit-def $eflags, implicit $mxcsr VCOMISDZrm_Int $xmm0, $rdi, 1, $noreg, 0, $noreg, implicit-def $eflags, implicit $mxcsr ; CHECK: VCOMISDrr_Int $xmm0, $xmm1, implicit-def $eflags, implicit $mxcsr @@ -2335,137 +2335,137 @@ name: evex_z256_to_evex_test body: | bb.0: ; CHECK: VMOVAPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVAPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVAPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVAPDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVAPDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVAPDZ256rr $ymm16 - $ymm16 = VMOVAPDZ256rr $ymm16 + $ymm16 = VMOVAPDZ256rr $ymm16 ; CHECK: VMOVAPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVAPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVAPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVAPSZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVAPSZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVAPSZ256rr $ymm16 - $ymm16 = VMOVAPSZ256rr $ymm16 + $ymm16 = VMOVAPSZ256rr $ymm16 ; CHECK: $ymm16 = VMOVDDUPZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVDDUPZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDDUPZ256rr $ymm16 - $ymm16 = VMOVDDUPZ256rr $ymm16 + $ymm16 = VMOVDDUPZ256rr $ymm16 ; CHECK: VMOVDQA32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVDQA32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVDQA32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVDQA32Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVDQA32Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDQA32Z256rr $ymm16 - $ymm16 = VMOVDQA32Z256rr $ymm16 + $ymm16 = VMOVDQA32Z256rr $ymm16 ; CHECK: VMOVDQA64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVDQA64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVDQA64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVDQA64Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVDQA64Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDQA64Z256rr $ymm16 - $ymm16 = VMOVDQA64Z256rr $ymm16 + $ymm16 = VMOVDQA64Z256rr $ymm16 ; CHECK: VMOVDQU16Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVDQU16Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVDQU16Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVDQU16Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVDQU16Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDQU16Z256rr $ymm16 - $ymm16 = VMOVDQU16Z256rr $ymm16 + $ymm16 = VMOVDQU16Z256rr $ymm16 ; CHECK: VMOVDQU32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVDQU32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVDQU32Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVDQU32Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVDQU32Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDQU32Z256rr $ymm16 - $ymm16 = VMOVDQU32Z256rr $ymm16 + $ymm16 = VMOVDQU32Z256rr $ymm16 ; CHECK: VMOVDQU64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVDQU64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVDQU64Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVDQU64Z256rm $rip, 1, $noreg, 0, $noreg - $ymm16 = VMOVDQU64Z256rm $rip, 1, $noreg, 0, $noreg + $ymm16 = VMOVDQU64Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDQU64Z256rr $ymm16 - $ymm16 = VMOVDQU64Z256rr $ymm16 + $ymm16 = VMOVDQU64Z256rr $ymm16 ; CHECK: VMOVDQU8Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVDQU8Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVDQU8Z256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVDQU8Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVDQU8Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVDQU8Z256rr $ymm16 - $ymm16 = VMOVDQU8Z256rr $ymm16 + $ymm16 = VMOVDQU8Z256rr $ymm16 ; CHECK: $ymm16 = VMOVNTDQAZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVNTDQAZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: VMOVNTDQZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVNTDQZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVNTDQZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: VMOVNTPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVNTPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVNTPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: VMOVNTPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVNTPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVNTPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVSHDUPZ256rm $rip, 1, $noreg, 0, $noreg - $ymm16 = VMOVSHDUPZ256rm $rip, 1, $noreg, 0, $noreg + $ymm16 = VMOVSHDUPZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVSHDUPZ256rr $ymm16 - $ymm16 = VMOVSHDUPZ256rr $ymm16 + $ymm16 = VMOVSHDUPZ256rr $ymm16 ; CHECK: $ymm16 = VMOVSLDUPZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVSLDUPZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVSLDUPZ256rr $ymm16 - $ymm16 = VMOVSLDUPZ256rr $ymm16 + $ymm16 = VMOVSLDUPZ256rr $ymm16 ; CHECK: VMOVUPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVUPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVUPDZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VMOVUPDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VMOVUPDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVUPDZ256rr $ymm16 - $ymm16 = VMOVUPDZ256rr $ymm16 - ; CHECK: VMOVUPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 - VMOVUPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + $ymm16 = VMOVUPDZ256rr $ymm16 + ; CHECK: VMOVUPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 + VMOVUPSZ256mr $rdi, 1, $noreg, 0, $noreg, $ymm16 ; CHECK: $ymm16 = VPANDDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPANDDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPANDDZ256rr $ymm16, $ymm1 - $ymm16 = VPANDDZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPANDDZ256rr $ymm16, $ymm1 + $ymm16 = VPANDDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPANDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPANDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPANDQZ256rr $ymm16, $ymm1 - $ymm16 = VPANDQZ256rr $ymm16, $ymm1 + $ymm16 = VPANDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPANDNDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPANDNDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPANDNDZ256rr $ymm16, $ymm1 - $ymm16 = VPANDNDZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPANDNDZ256rr $ymm16, $ymm1 + $ymm16 = VPANDNDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPANDNQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPANDNQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPANDNQZ256rr $ymm16, $ymm1 - $ymm16 = VPANDNQZ256rr $ymm16, $ymm1 + $ymm16 = VPANDNQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPAVGBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPAVGBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPAVGBZ256rr $ymm16, $ymm1 - $ymm16 = VPAVGBZ256rr $ymm16, $ymm1 + $ymm16 = VPAVGBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPAVGWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPAVGWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPAVGWZ256rr $ymm16, $ymm1 - $ymm16 = VPAVGWZ256rr $ymm16, $ymm1 + $ymm16 = VPAVGWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPADDBZ256rr $ymm16, $ymm1 - $ymm16 = VPADDBZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPADDBZ256rr $ymm16, $ymm1 + $ymm16 = VPADDBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDDZ256rr $ymm16, $ymm1 - $ymm16 = VPADDDZ256rr $ymm16, $ymm1 + $ymm16 = VPADDDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDQZ256rr $ymm16, $ymm1 - $ymm16 = VPADDQZ256rr $ymm16, $ymm1 + $ymm16 = VPADDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDSBZ256rr $ymm16, $ymm1 - $ymm16 = VPADDSBZ256rr $ymm16, $ymm1 + $ymm16 = VPADDSBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDSWZ256rr $ymm16, $ymm1 - $ymm16 = VPADDSWZ256rr $ymm16, $ymm1 + $ymm16 = VPADDSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDUSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDUSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDUSBZ256rr $ymm16, $ymm1 - $ymm16 = VPADDUSBZ256rr $ymm16, $ymm1 + $ymm16 = VPADDUSBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDUSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDUSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDUSWZ256rr $ymm16, $ymm1 - $ymm16 = VPADDUSWZ256rr $ymm16, $ymm1 + $ymm16 = VPADDUSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPADDWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPADDWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPADDWZ256rr $ymm16, $ymm1 - $ymm16 = VPADDWZ256rr $ymm16, $ymm1 + $ymm16 = VPADDWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VMULPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm16 = VMULPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm16 = VMULPDZ256rr $ymm16, $ymm1, implicit $mxcsr @@ -2477,143 +2477,143 @@ body: | ; CHECK: $ymm16 = VORPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VORPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VORPDZ256rr $ymm16, $ymm1 - $ymm16 = VORPDZ256rr $ymm16, $ymm1 + $ymm16 = VORPDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VORPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VORPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VORPSZ256rr $ymm16, $ymm1 - $ymm16 = VORPSZ256rr $ymm16, $ymm1 + $ymm16 = VORPSZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMADDUBSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMADDUBSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMADDUBSWZ256rr $ymm16, $ymm1 - $ymm16 = VPMADDUBSWZ256rr $ymm16, $ymm1 + $ymm16 = VPMADDUBSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMADDWDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMADDWDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMADDWDZ256rr $ymm16, $ymm1 - $ymm16 = VPMADDWDZ256rr $ymm16, $ymm1 + $ymm16 = VPMADDWDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMAXSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMAXSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMAXSBZ256rr $ymm16, $ymm1 - $ymm16 = VPMAXSBZ256rr $ymm16, $ymm1 + $ymm16 = VPMAXSBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMAXSDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMAXSDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMAXSDZ256rr $ymm16, $ymm1 - $ymm16 = VPMAXSDZ256rr $ymm16, $ymm1 + $ymm16 = VPMAXSDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMAXSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMAXSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMAXSWZ256rr $ymm16, $ymm1 - $ymm16 = VPMAXSWZ256rr $ymm16, $ymm1 + $ymm16 = VPMAXSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMAXUBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMAXUBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMAXUBZ256rr $ymm16, $ymm1 - $ymm16 = VPMAXUBZ256rr $ymm16, $ymm1 + $ymm16 = VPMAXUBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMAXUDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMAXUDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMAXUDZ256rr $ymm16, $ymm1 - $ymm16 = VPMAXUDZ256rr $ymm16, $ymm1 + $ymm16 = VPMAXUDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMAXUWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMAXUWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMAXUWZ256rr $ymm16, $ymm1 - $ymm16 = VPMAXUWZ256rr $ymm16, $ymm1 + $ymm16 = VPMAXUWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMINSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMINSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMINSBZ256rr $ymm16, $ymm1 - $ymm16 = VPMINSBZ256rr $ymm16, $ymm1 + $ymm16 = VPMINSBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMINSDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMINSDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMINSDZ256rr $ymm16, $ymm1 - $ymm16 = VPMINSDZ256rr $ymm16, $ymm1 + $ymm16 = VPMINSDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMINSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMINSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMINSWZ256rr $ymm16, $ymm1 - $ymm16 = VPMINSWZ256rr $ymm16, $ymm1 + $ymm16 = VPMINSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMINUBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMINUBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMINUBZ256rr $ymm16, $ymm1 - $ymm16 = VPMINUBZ256rr $ymm16, $ymm1 + $ymm16 = VPMINUBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMINUDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMINUDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMINUDZ256rr $ymm16, $ymm1 - $ymm16 = VPMINUDZ256rr $ymm16, $ymm1 + $ymm16 = VPMINUDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMINUWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMINUWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMINUWZ256rr $ymm16, $ymm1 - $ymm16 = VPMINUWZ256rr $ymm16, $ymm1 + $ymm16 = VPMINUWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPMULDQZ256rr $ymm16, $ymm1 - $ymm16 = VPMULDQZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPMULDQZ256rr $ymm16, $ymm1 + $ymm16 = VPMULDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULHRSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULHRSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMULHRSWZ256rr $ymm16, $ymm1 - $ymm16 = VPMULHRSWZ256rr $ymm16, $ymm1 + $ymm16 = VPMULHRSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULHUWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULHUWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMULHUWZ256rr $ymm16, $ymm1 - $ymm16 = VPMULHUWZ256rr $ymm16, $ymm1 + $ymm16 = VPMULHUWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULHWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULHWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMULHWZ256rr $ymm16, $ymm1 - $ymm16 = VPMULHWZ256rr $ymm16, $ymm1 + $ymm16 = VPMULHWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULLDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULLDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMULLDZ256rr $ymm16, $ymm1 - $ymm16 = VPMULLDZ256rr $ymm16, $ymm1 + $ymm16 = VPMULLDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULLWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULLWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPMULLWZ256rr $ymm16, $ymm1 - $ymm16 = VPMULLWZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPMULLWZ256rr $ymm16, $ymm1 + $ymm16 = VPMULLWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPMULUDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPMULUDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMULUDQZ256rr $ymm16, $ymm1 - $ymm16 = VPMULUDQZ256rr $ymm16, $ymm1 + $ymm16 = VPMULUDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPORDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPORDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPORDZ256rr $ymm16, $ymm1 - $ymm16 = VPORDZ256rr $ymm16, $ymm1 + $ymm16 = VPORDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPORQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPORQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPORQZ256rr $ymm16, $ymm1 - $ymm16 = VPORQZ256rr $ymm16, $ymm1 + $ymm16 = VPORQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBBZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBBZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBDZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBDZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBQZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBQZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBSBZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBSBZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBSBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBSWZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBSWZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBUSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBUSBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBUSBZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBUSBZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBUSBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBUSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBUSWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBUSWZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBUSWZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBUSWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSUBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSUBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSUBWZ256rr $ymm16, $ymm1 - $ymm16 = VPSUBWZ256rr $ymm16, $ymm1 + $ymm16 = VPSUBWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPXORDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPXORDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPXORDZ256rr $ymm16, $ymm1 - $ymm16 = VPXORDZ256rr $ymm16, $ymm1 + $ymm16 = VPXORDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPXORQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPXORQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPXORQZ256rr $ymm16, $ymm1 - $ymm16 = VPXORQZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPXORQZ256rr $ymm16, $ymm1 + $ymm16 = VPXORQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VADDPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm16 = VADDPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm16 = VADDPDZ256rr $ymm16, $ymm1, implicit $mxcsr @@ -2625,19 +2625,19 @@ body: | ; CHECK: $ymm16 = VANDNPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VANDNPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VANDNPDZ256rr $ymm16, $ymm1 - $ymm16 = VANDNPDZ256rr $ymm16, $ymm1 + $ymm16 = VANDNPDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VANDNPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VANDNPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VANDNPSZ256rr $ymm16, $ymm1 - $ymm16 = VANDNPSZ256rr $ymm16, $ymm1 + $ymm16 = VANDNPSZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VANDPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VANDPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VANDPDZ256rr $ymm16, $ymm1 - $ymm16 = VANDPDZ256rr $ymm16, $ymm1 + $ymm16 = VANDPDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VANDPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VANDPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VANDPSZ256rr $ymm16, $ymm1 - $ymm16 = VANDPSZ256rr $ymm16, $ymm1 + $ymm16 = VANDPSZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VDIVPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm16 = VDIVPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm16 = VDIVPDZ256rr $ymm16, $ymm1, implicit $mxcsr @@ -2681,43 +2681,43 @@ body: | ; CHECK: $ymm16 = VXORPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VXORPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VXORPDZ256rr $ymm16, $ymm1 - $ymm16 = VXORPDZ256rr $ymm16, $ymm1 + $ymm16 = VXORPDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VXORPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VXORPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VXORPSZ256rr $ymm16, $ymm1 - $ymm16 = VXORPSZ256rr $ymm16, $ymm1 + $ymm16 = VXORPSZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPACKSSDWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPACKSSDWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPACKSSDWZ256rr $ymm16, $ymm1 - $ymm16 = VPACKSSDWZ256rr $ymm16, $ymm1 + $ymm16 = VPACKSSDWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPACKSSWBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPACKSSWBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPACKSSWBZ256rr $ymm16, $ymm1 - $ymm16 = VPACKSSWBZ256rr $ymm16, $ymm1 + $ymm16 = VPACKSSWBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPACKUSDWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPACKUSDWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPACKUSDWZ256rr $ymm16, $ymm1 - $ymm16 = VPACKUSDWZ256rr $ymm16, $ymm1 + $ymm16 = VPACKUSDWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPACKUSWBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPACKUSWBZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPACKUSWBZ256rr $ymm16, $ymm1 - $ymm16 = VPACKUSWBZ256rr $ymm16, $ymm1 + $ymm16 = VPACKUSWBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VUNPCKHPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VUNPCKHPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VUNPCKHPDZ256rr $ymm16, $ymm1 - $ymm16 = VUNPCKHPDZ256rr $ymm16, $ymm1 + $ymm16 = VUNPCKHPDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VUNPCKHPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VUNPCKHPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VUNPCKHPSZ256rr $ymm16, $ymm1 - $ymm16 = VUNPCKHPSZ256rr $ymm16, $ymm1 + $ymm16 = VUNPCKHPSZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VUNPCKLPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VUNPCKLPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VUNPCKLPDZ256rr $ymm16, $ymm1 - $ymm16 = VUNPCKLPDZ256rr $ymm16, $ymm1 + $ymm16 = VUNPCKLPDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VUNPCKLPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VUNPCKLPSZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VUNPCKLPSZ256rr $ymm16, $ymm1 - $ymm16 = VUNPCKLPSZ256rr $ymm16, $ymm1 + $ymm16 = VUNPCKLPSZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VSUBPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm16 = VSUBPDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm16 = VSUBPDZ256rr $ymm16, $ymm1, implicit $mxcsr @@ -2729,35 +2729,35 @@ body: | ; CHECK: $ymm16 = VPUNPCKHBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKHBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPUNPCKHBWZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKHBWZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKHBWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKHDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKHDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPUNPCKHDQZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKHDQZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKHDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKHQDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKHQDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPUNPCKHQDQZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKHQDQZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKHQDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKHWDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKHWDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPUNPCKHWDZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKHWDZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKHWDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKLBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKLBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPUNPCKLBWZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKLBWZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKLBWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKLDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKLDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPUNPCKLDQZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKLDQZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPUNPCKLDQZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKLDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKLQDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKLQDQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPUNPCKLQDQZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKLQDQZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPUNPCKLQDQZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKLQDQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPUNPCKLWDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPUNPCKLWDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPUNPCKLWDZ256rr $ymm16, $ymm1 - $ymm16 = VPUNPCKLWDZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPUNPCKLWDZ256rr $ymm16, $ymm1 + $ymm16 = VPUNPCKLWDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VFMADD132PDZ256m $ymm16, $ymm16, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr $ymm16 = VFMADD132PDZ256m $ymm16, $ymm16, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $ymm16 = VFMADD132PDZ256r $ymm16, $ymm1, $ymm2, implicit $mxcsr @@ -2903,97 +2903,97 @@ body: | ; CHECK: $ymm16 = VFNMSUB231PSZ256r $ymm16, $ymm1, $ymm2, implicit $mxcsr $ymm16 = VFNMSUB231PSZ256r $ymm16, $ymm1, $ymm2, implicit $mxcsr ; CHECK: $ymm16 = VPSRADZ256ri $ymm16, 7 - $ymm16 = VPSRADZ256ri $ymm16, 7 + $ymm16 = VPSRADZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSRADZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRADZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRADZ256rr $ymm16, $xmm1 - $ymm16 = VPSRADZ256rr $ymm16, $xmm1 + $ymm16 = VPSRADZ256rr $ymm16, $xmm1 ; CHECK: $ymm16 = VPSRAVDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRAVDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRAVDZ256rr $ymm16, $ymm1 - $ymm16 = VPSRAVDZ256rr $ymm16, $ymm1 + $ymm16 = VPSRAVDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSRAWZ256ri $ymm16, 7 - $ymm16 = VPSRAWZ256ri $ymm16, 7 + $ymm16 = VPSRAWZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSRAWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRAWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRAWZ256rr $ymm16, $xmm1 - $ymm16 = VPSRAWZ256rr $ymm16, $xmm1 + $ymm16 = VPSRAWZ256rr $ymm16, $xmm1 ; CHECK: $ymm16 = VPSRLDQZ256ri $ymm16, 7 $ymm16 = VPSRLDQZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSRLDZ256ri $ymm16, 7 - $ymm16 = VPSRLDZ256ri $ymm16, 7 + $ymm16 = VPSRLDZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSRLDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRLDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRLDZ256rr $ymm16, $xmm1 - $ymm16 = VPSRLDZ256rr $ymm16, $xmm1 + $ymm16 = VPSRLDZ256rr $ymm16, $xmm1 ; CHECK: $ymm16 = VPSRLQZ256ri $ymm16, 7 - $ymm16 = VPSRLQZ256ri $ymm16, 7 + $ymm16 = VPSRLQZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSRLQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRLQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRLQZ256rr $ymm16, $xmm1 - $ymm16 = VPSRLQZ256rr $ymm16, $xmm1 + $ymm16 = VPSRLQZ256rr $ymm16, $xmm1 ; CHECK: $ymm16 = VPSRLVDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRLVDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRLVDZ256rr $ymm16, $ymm1 - $ymm16 = VPSRLVDZ256rr $ymm16, $ymm1 + $ymm16 = VPSRLVDZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSRLVQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRLVQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSRLVQZ256rr $ymm16, $ymm1 - $ymm16 = VPSRLVQZ256rr $ymm16, $ymm1 + $ymm16 = VPSRLVQZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSRLWZ256ri $ymm16, 7 - $ymm16 = VPSRLWZ256ri $ymm16, 7 + $ymm16 = VPSRLWZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSRLWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSRLWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPSRLWZ256rr $ymm16, $xmm1 - $ymm16 = VPSRLWZ256rr $ymm16, $xmm1 + ; CHECK: $ymm16 = VPSRLWZ256rr $ymm16, $xmm1 + $ymm16 = VPSRLWZ256rr $ymm16, $xmm1 ; CHECK: $ymm16 = VPMOVSXBDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVSXBDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVSXBDZ256rr $xmm0 - $ymm16 = VPMOVSXBDZ256rr $xmm0 + $ymm16 = VPMOVSXBDZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVSXBQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVSXBQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVSXBQZ256rr $xmm0 - $ymm16 = VPMOVSXBQZ256rr $xmm0 + $ymm16 = VPMOVSXBQZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVSXBWZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVSXBWZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVSXBWZ256rr $xmm0 - $ymm16 = VPMOVSXBWZ256rr $xmm0 + $ymm16 = VPMOVSXBWZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVSXDQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVSXDQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVSXDQZ256rr $xmm0 - $ymm16 = VPMOVSXDQZ256rr $xmm0 + $ymm16 = VPMOVSXDQZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVSXWDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVSXWDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVSXWDZ256rr $xmm0 - $ymm16 = VPMOVSXWDZ256rr $xmm0 + $ymm16 = VPMOVSXWDZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVSXWQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVSXWQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVSXWQZ256rr $xmm0 - $ymm16 = VPMOVSXWQZ256rr $xmm0 + $ymm16 = VPMOVSXWQZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVZXBDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVZXBDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVZXBDZ256rr $xmm0 - $ymm16 = VPMOVZXBDZ256rr $xmm0 + $ymm16 = VPMOVZXBDZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVZXBQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVZXBQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVZXBQZ256rr $xmm0 - $ymm16 = VPMOVZXBQZ256rr $xmm0 + $ymm16 = VPMOVZXBQZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVZXBWZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVZXBWZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVZXBWZ256rr $xmm0 - $ymm16 = VPMOVZXBWZ256rr $xmm0 + $ymm16 = VPMOVZXBWZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVZXDQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVZXDQZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVZXDQZ256rr $xmm0 - $ymm16 = VPMOVZXDQZ256rr $xmm0 + $ymm16 = VPMOVZXDQZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVZXWDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVZXWDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPMOVZXWDZ256rr $xmm0 - $ymm16 = VPMOVZXWDZ256rr $xmm0 + $ymm16 = VPMOVZXWDZ256rr $xmm0 ; CHECK: $ymm16 = VPMOVZXWQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPMOVZXWQZ256rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPMOVZXWQZ256rr $xmm0 - $ymm16 = VPMOVZXWQZ256rr $xmm0 + ; CHECK: $ymm16 = VPMOVZXWQZ256rr $xmm0 + $ymm16 = VPMOVZXWQZ256rr $xmm0 ; CHECK: $ymm16 = VBROADCASTF32X2Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VBROADCASTF32X2Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VBROADCASTF32X2Z256rr $xmm16 @@ -3003,23 +3003,23 @@ body: | ; CHECK: $ymm16 = VBROADCASTSDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VBROADCASTSDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VBROADCASTSDZ256rr $xmm0 - $ymm16 = VBROADCASTSDZ256rr $xmm0 + $ymm16 = VBROADCASTSDZ256rr $xmm0 ; CHECK: $ymm16 = VBROADCASTSSZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VBROADCASTSSZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VBROADCASTSSZ256rr $xmm0 - $ymm16 = VBROADCASTSSZ256rr $xmm0 + $ymm16 = VBROADCASTSSZ256rr $xmm0 ; CHECK: $ymm16 = VPBROADCASTBZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPBROADCASTBZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPBROADCASTBZ256rr $xmm0 - $ymm16 = VPBROADCASTBZ256rr $xmm0 + $ymm16 = VPBROADCASTBZ256rr $xmm0 ; CHECK: $ymm16 = VPBROADCASTDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPBROADCASTDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPBROADCASTDZ256rr $xmm0 - $ymm16 = VPBROADCASTDZ256rr $xmm0 + $ymm16 = VPBROADCASTDZ256rr $xmm0 ; CHECK: $ymm16 = VPBROADCASTWZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPBROADCASTWZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPBROADCASTWZ256rr $xmm0 - $ymm16 = VPBROADCASTWZ256rr $xmm0 + $ymm16 = VPBROADCASTWZ256rr $xmm0 ; CHECK: $ymm16 = VBROADCASTI32X4Z256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VBROADCASTI32X4Z256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VBROADCASTI32X2Z256rm $rip, 1, $noreg, 0, $noreg @@ -3028,66 +3028,66 @@ body: | $ymm16 = VBROADCASTI32X2Z256rr $xmm16 ; CHECK: $ymm16 = VPBROADCASTQZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPBROADCASTQZ256rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPBROADCASTQZ256rr $xmm0 - $ymm16 = VPBROADCASTQZ256rr $xmm0 + ; CHECK: $ymm16 = VPBROADCASTQZ256rr $xmm0 + $ymm16 = VPBROADCASTQZ256rr $xmm0 ; CHECK: $ymm16 = VPABSBZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPABSBZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPABSBZ256rr $ymm16 - $ymm16 = VPABSBZ256rr $ymm16 + $ymm16 = VPABSBZ256rr $ymm16 ; CHECK: $ymm16 = VPABSDZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPABSDZ256rm $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPABSDZ256rr $ymm16 - $ymm16 = VPABSDZ256rr $ymm16 + $ymm16 = VPABSDZ256rr $ymm16 ; CHECK: $ymm16 = VPABSWZ256rm $rip, 1, $noreg, 0, $noreg $ymm16 = VPABSWZ256rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPABSWZ256rr $ymm16 - $ymm16 = VPABSWZ256rr $ymm16 + ; CHECK: $ymm16 = VPABSWZ256rr $ymm16 + $ymm16 = VPABSWZ256rr $ymm16 ; CHECK: $ymm16 = VPSADBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSADBWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $ymm16 = VPSADBWZ256rr $ymm16, $ymm1 - $ymm16 = VPSADBWZ256rr $ymm16, $ymm1 + ; CHECK: $ymm16 = VPSADBWZ256rr $ymm16, $ymm1 + $ymm16 = VPSADBWZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPERMDZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg - $ymm16 = VPERMDZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg + $ymm16 = VPERMDZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPERMDZ256rr $ymm1, $ymm16 - $ymm16 = VPERMDZ256rr $ymm1, $ymm16 + $ymm16 = VPERMDZ256rr $ymm1, $ymm16 ; CHECK: $ymm16 = VPERMILPDZ256mi $rdi, 1, $noreg, 0, $noreg, 7 $ymm16 = VPERMILPDZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm16 = VPERMILPDZ256ri $ymm16, 7 - $ymm16 = VPERMILPDZ256ri $ymm16, 7 + $ymm16 = VPERMILPDZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPERMILPDZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg - $ymm16 = VPERMILPDZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg + $ymm16 = VPERMILPDZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPERMILPDZ256rr $ymm1, $ymm16 - $ymm16 = VPERMILPDZ256rr $ymm1, $ymm16 + $ymm16 = VPERMILPDZ256rr $ymm1, $ymm16 ; CHECK: $ymm16 = VPERMILPSZ256mi $rdi, 1, $noreg, 0, $noreg, 7 $ymm16 = VPERMILPSZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm16 = VPERMILPSZ256ri $ymm16, 7 - $ymm16 = VPERMILPSZ256ri $ymm16, 7 + $ymm16 = VPERMILPSZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPERMILPSZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg - $ymm16 = VPERMILPSZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg + $ymm16 = VPERMILPSZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPERMILPSZ256rr $ymm1, $ymm16 - $ymm16 = VPERMILPSZ256rr $ymm1, $ymm16 + $ymm16 = VPERMILPSZ256rr $ymm1, $ymm16 ; CHECK: $ymm16 = VPERMPDZ256mi $rdi, 1, $noreg, 0, $noreg, 7 $ymm16 = VPERMPDZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm16 = VPERMPDZ256ri $ymm16, 7 - $ymm16 = VPERMPDZ256ri $ymm16, 7 + $ymm16 = VPERMPDZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPERMPSZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg - $ymm16 = VPERMPSZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg + $ymm16 = VPERMPSZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPERMPSZ256rr $ymm1, $ymm16 - $ymm16 = VPERMPSZ256rr $ymm1, $ymm16 + $ymm16 = VPERMPSZ256rr $ymm1, $ymm16 ; CHECK: $ymm16 = VPERMQZ256mi $rdi, 1, $noreg, 0, $noreg, 7 $ymm16 = VPERMQZ256mi $rdi, 1, $noreg, 0, $noreg, 7 ; CHECK: $ymm16 = VPERMQZ256ri $ymm16, 7 - $ymm16 = VPERMQZ256ri $ymm16, 7 + $ymm16 = VPERMQZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSLLDQZ256ri $ymm16, 14 $ymm16 = VPSLLDQZ256ri $ymm16, 14 ; CHECK: $ymm16 = VPSLLDZ256ri $ymm16, 7 - $ymm16 = VPSLLDZ256ri $ymm16, 7 + $ymm16 = VPSLLDZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSLLDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSLLDZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSLLDZ256rr $ymm16, $xmm16 $ymm16 = VPSLLDZ256rr $ymm16, $xmm16 ; CHECK: $ymm16 = VPSLLQZ256ri $ymm16, 7 - $ymm16 = VPSLLQZ256ri $ymm16, 7 + $ymm16 = VPSLLQZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSLLQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSLLQZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSLLQZ256rr $ymm16, $xmm16 @@ -3101,7 +3101,7 @@ body: | ; CHECK: $ymm16 = VPSLLVQZ256rr $ymm16, $ymm16 $ymm16 = VPSLLVQZ256rr $ymm16, $ymm16 ; CHECK: $ymm16 = VPSLLWZ256ri $ymm16, 7 - $ymm16 = VPSLLWZ256ri $ymm16, 7 + $ymm16 = VPSLLWZ256ri $ymm16, 7 ; CHECK: $ymm16 = VPSLLWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg $ymm16 = VPSLLWZ256rm $ymm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSLLWZ256rr $ymm16, $xmm16 @@ -3158,26 +3158,26 @@ body: | $ymm16 = VPALIGNRZ256rmi $ymm16, $rdi, 1, $noreg, 0, $noreg, 1 ; CHECK: $ymm16 = VPALIGNRZ256rri $ymm16, $ymm1, 1 $ymm16 = VPALIGNRZ256rri $ymm16, $ymm1, 1 - ; CHECK: $ymm16 = VMOVUPSZ256rm $rdi, 1, $noreg, 0, $noreg - $ymm16 = VMOVUPSZ256rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $ymm16 = VMOVUPSZ256rm $rdi, 1, $noreg, 0, $noreg + $ymm16 = VMOVUPSZ256rm $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VMOVUPSZ256rr $ymm16 - $ymm16 = VMOVUPSZ256rr $ymm16 + $ymm16 = VMOVUPSZ256rr $ymm16 ; CHECK: $ymm16 = VPSHUFBZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg $ymm16 = VPSHUFBZ256rm $ymm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $ymm16 = VPSHUFBZ256rr $ymm16, $ymm1 - $ymm16 = VPSHUFBZ256rr $ymm16, $ymm1 + $ymm16 = VPSHUFBZ256rr $ymm16, $ymm1 ; CHECK: $ymm16 = VPSHUFDZ256mi $rdi, 1, $noreg, 0, $noreg, -24 $ymm16 = VPSHUFDZ256mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm16 = VPSHUFDZ256ri $ymm16, -24 - $ymm16 = VPSHUFDZ256ri $ymm16, -24 + $ymm16 = VPSHUFDZ256ri $ymm16, -24 ; CHECK: $ymm16 = VPSHUFHWZ256mi $rdi, 1, $noreg, 0, $noreg, -24 $ymm16 = VPSHUFHWZ256mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm16 = VPSHUFHWZ256ri $ymm16, -24 - $ymm16 = VPSHUFHWZ256ri $ymm16, -24 + $ymm16 = VPSHUFHWZ256ri $ymm16, -24 ; CHECK: $ymm16 = VPSHUFLWZ256mi $rdi, 1, $noreg, 0, $noreg, -24 $ymm16 = VPSHUFLWZ256mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm16 = VPSHUFLWZ256ri $ymm16, -24 - $ymm16 = VPSHUFLWZ256ri $ymm16, -24 + $ymm16 = VPSHUFLWZ256ri $ymm16, -24 ; CHECK: $ymm16 = VSHUFPDZ256rmi $ymm16, $rip, 1, $noreg, 0, $noreg, -24 $ymm16 = VSHUFPDZ256rmi $ymm16, $rip, 1, $noreg, 0, $noreg, -24 ; CHECK: $ymm16 = VSHUFPDZ256rri $ymm16, $ymm1, -24 @@ -3220,8 +3220,8 @@ body: | $ymm16 = VSHUFI64X2Z256rri $ymm16, $ymm1, 228 RET64 -... ---- +... +--- # CHECK-LABEL: name: evex_z128_to_evex_test # CHECK: bb.0: @@ -3229,137 +3229,137 @@ name: evex_z128_to_evex_test body: | bb.0: ; CHECK: VMOVAPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVAPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVAPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVAPDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVAPDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVAPDZ128rr $xmm16 - $xmm16 = VMOVAPDZ128rr $xmm16 + $xmm16 = VMOVAPDZ128rr $xmm16 ; CHECK: VMOVAPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVAPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVAPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVAPSZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVAPSZ128rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VMOVAPSZ128rr $xmm16 - $xmm16 = VMOVAPSZ128rr $xmm16 + ; CHECK: $xmm16 = VMOVAPSZ128rr $xmm16 + $xmm16 = VMOVAPSZ128rr $xmm16 ; CHECK: VMOVDQA32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVDQA32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVDQA32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVDQA32Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDQA32Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDQA32Z128rr $xmm16 - $xmm16 = VMOVDQA32Z128rr $xmm16 + $xmm16 = VMOVDQA32Z128rr $xmm16 ; CHECK: VMOVDQA64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVDQA64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVDQA64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVDQA64Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDQA64Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDQA64Z128rr $xmm16 - $xmm16 = VMOVDQA64Z128rr $xmm16 + $xmm16 = VMOVDQA64Z128rr $xmm16 ; CHECK: VMOVDQU16Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVDQU16Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVDQU16Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVDQU16Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDQU16Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDQU16Z128rr $xmm16 - $xmm16 = VMOVDQU16Z128rr $xmm16 + $xmm16 = VMOVDQU16Z128rr $xmm16 ; CHECK: VMOVDQU32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVDQU32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVDQU32Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVDQU32Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDQU32Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDQU32Z128rr $xmm16 - $xmm16 = VMOVDQU32Z128rr $xmm16 + $xmm16 = VMOVDQU32Z128rr $xmm16 ; CHECK: VMOVDQU64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVDQU64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVDQU64Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVDQU64Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDQU64Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDQU64Z128rr $xmm16 - $xmm16 = VMOVDQU64Z128rr $xmm16 + $xmm16 = VMOVDQU64Z128rr $xmm16 ; CHECK: VMOVDQU8Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVDQU8Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVDQU8Z128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVDQU8Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDQU8Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDQU8Z128rr $xmm16 - $xmm16 = VMOVDQU8Z128rr $xmm16 + $xmm16 = VMOVDQU8Z128rr $xmm16 ; CHECK: $xmm16 = VMOVNTDQAZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVNTDQAZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: VMOVUPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVUPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVUPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVUPDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVUPDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVUPDZ128rr $xmm16 - $xmm16 = VMOVUPDZ128rr $xmm16 + $xmm16 = VMOVUPDZ128rr $xmm16 ; CHECK: VMOVUPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVUPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVUPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVUPSZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVUPSZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVUPSZ128rr $xmm16 - $xmm16 = VMOVUPSZ128rr $xmm16 + $xmm16 = VMOVUPSZ128rr $xmm16 ; CHECK: VMOVNTDQZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVNTDQZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVNTDQZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: VMOVNTPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVNTPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVNTPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: VMOVNTPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVNTPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVNTPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VPMOVSXBDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVSXBDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVSXBDZ128rr $xmm16 - $xmm16 = VPMOVSXBDZ128rr $xmm16 + $xmm16 = VPMOVSXBDZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVSXBQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVSXBQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVSXBQZ128rr $xmm16 - $xmm16 = VPMOVSXBQZ128rr $xmm16 + $xmm16 = VPMOVSXBQZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVSXBWZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVSXBWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVSXBWZ128rr $xmm16 - $xmm16 = VPMOVSXBWZ128rr $xmm16 + $xmm16 = VPMOVSXBWZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVSXDQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVSXDQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVSXDQZ128rr $xmm16 - $xmm16 = VPMOVSXDQZ128rr $xmm16 + $xmm16 = VPMOVSXDQZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVSXWDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVSXWDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVSXWDZ128rr $xmm16 - $xmm16 = VPMOVSXWDZ128rr $xmm16 + $xmm16 = VPMOVSXWDZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVSXWQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVSXWQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVSXWQZ128rr $xmm16 - $xmm16 = VPMOVSXWQZ128rr $xmm16 + $xmm16 = VPMOVSXWQZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVZXBDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVZXBDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVZXBDZ128rr $xmm16 - $xmm16 = VPMOVZXBDZ128rr $xmm16 + $xmm16 = VPMOVZXBDZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVZXBQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVZXBQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVZXBQZ128rr $xmm16 - $xmm16 = VPMOVZXBQZ128rr $xmm16 + $xmm16 = VPMOVZXBQZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVZXBWZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVZXBWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVZXBWZ128rr $xmm16 - $xmm16 = VPMOVZXBWZ128rr $xmm16 + $xmm16 = VPMOVZXBWZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVZXDQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVZXDQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVZXDQZ128rr $xmm16 - $xmm16 = VPMOVZXDQZ128rr $xmm16 + $xmm16 = VPMOVZXDQZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVZXWDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVZXWDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMOVZXWDZ128rr $xmm16 - $xmm16 = VPMOVZXWDZ128rr $xmm16 + $xmm16 = VPMOVZXWDZ128rr $xmm16 ; CHECK: $xmm16 = VPMOVZXWQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPMOVZXWQZ128rm $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPMOVZXWQZ128rr $xmm16 - $xmm16 = VPMOVZXWQZ128rr $xmm16 + ; CHECK: $xmm16 = VPMOVZXWQZ128rr $xmm16 + $xmm16 = VPMOVZXWQZ128rr $xmm16 ; CHECK: VMOVHPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVHPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVHPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVHPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVHPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVHPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: VMOVHPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVHPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVHPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVHPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVHPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVHPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: VMOVLPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVLPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + VMOVLPDZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVLPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVLPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVLPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: VMOVLPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - VMOVLPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 - ; CHECK: $xmm16 = VMOVLPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVLPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + VMOVLPSZ128mr $rdi, 1, $noreg, 0, $noreg, $xmm16 + ; CHECK: $xmm16 = VMOVLPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVLPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMAXCPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VMAXCPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VMAXCPDZ128rr $xmm16, $xmm1, implicit $mxcsr @@ -3403,183 +3403,183 @@ body: | ; CHECK: $xmm16 = VORPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VORPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VORPDZ128rr $xmm16, $xmm1 - $xmm16 = VORPDZ128rr $xmm16, $xmm1 + $xmm16 = VORPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VORPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VORPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VORPSZ128rr $xmm16, $xmm1 - $xmm16 = VORPSZ128rr $xmm16, $xmm1 + $xmm16 = VORPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDBZ128rr $xmm16, $xmm1 - $xmm16 = VPADDBZ128rr $xmm16, $xmm1 + $xmm16 = VPADDBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDDZ128rr $xmm16, $xmm1 - $xmm16 = VPADDDZ128rr $xmm16, $xmm1 + $xmm16 = VPADDDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDQZ128rr $xmm16, $xmm1 - $xmm16 = VPADDQZ128rr $xmm16, $xmm1 + $xmm16 = VPADDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDSBZ128rr $xmm16, $xmm1 - $xmm16 = VPADDSBZ128rr $xmm16, $xmm1 + $xmm16 = VPADDSBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDSWZ128rr $xmm16, $xmm1 - $xmm16 = VPADDSWZ128rr $xmm16, $xmm1 + $xmm16 = VPADDSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDUSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDUSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDUSBZ128rr $xmm16, $xmm1 - $xmm16 = VPADDUSBZ128rr $xmm16, $xmm1 + $xmm16 = VPADDUSBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDUSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDUSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDUSWZ128rr $xmm16, $xmm1 - $xmm16 = VPADDUSWZ128rr $xmm16, $xmm1 + $xmm16 = VPADDUSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPADDWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPADDWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPADDWZ128rr $xmm16, $xmm1 - $xmm16 = VPADDWZ128rr $xmm16, $xmm1 + $xmm16 = VPADDWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPANDDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPANDDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPANDDZ128rr $xmm16, $xmm1 - $xmm16 = VPANDDZ128rr $xmm16, $xmm1 + $xmm16 = VPANDDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPANDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPANDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPANDQZ128rr $xmm16, $xmm1 - $xmm16 = VPANDQZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPANDQZ128rr $xmm16, $xmm1 + $xmm16 = VPANDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPANDNDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPANDNDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPANDNDZ128rr $xmm16, $xmm1 - $xmm16 = VPANDNDZ128rr $xmm16, $xmm1 + $xmm16 = VPANDNDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPANDNQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPANDNQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPANDNQZ128rr $xmm16, $xmm1 - $xmm16 = VPANDNQZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPANDNQZ128rr $xmm16, $xmm1 + $xmm16 = VPANDNQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPAVGBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPAVGBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPAVGBZ128rr $xmm16, $xmm1 - $xmm16 = VPAVGBZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPAVGBZ128rr $xmm16, $xmm1 + $xmm16 = VPAVGBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPAVGWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPAVGWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPAVGWZ128rr $xmm16, $xmm1 - $xmm16 = VPAVGWZ128rr $xmm16, $xmm1 + $xmm16 = VPAVGWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMAXSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMAXSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMAXSBZ128rr $xmm16, $xmm1 - $xmm16 = VPMAXSBZ128rr $xmm16, $xmm1 + $xmm16 = VPMAXSBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMAXSDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMAXSDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMAXSDZ128rr $xmm16, $xmm1 - $xmm16 = VPMAXSDZ128rr $xmm16, $xmm1 + $xmm16 = VPMAXSDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMAXSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMAXSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPMAXSWZ128rr $xmm16, $xmm1 - $xmm16 = VPMAXSWZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPMAXSWZ128rr $xmm16, $xmm1 + $xmm16 = VPMAXSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMAXUBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMAXUBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMAXUBZ128rr $xmm16, $xmm1 - $xmm16 = VPMAXUBZ128rr $xmm16, $xmm1 + $xmm16 = VPMAXUBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMAXUDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMAXUDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMAXUDZ128rr $xmm16, $xmm1 - $xmm16 = VPMAXUDZ128rr $xmm16, $xmm1 + $xmm16 = VPMAXUDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMAXUWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMAXUWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMAXUWZ128rr $xmm16, $xmm1 - $xmm16 = VPMAXUWZ128rr $xmm16, $xmm1 + $xmm16 = VPMAXUWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMINSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMINSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMINSBZ128rr $xmm16, $xmm1 - $xmm16 = VPMINSBZ128rr $xmm16, $xmm1 + $xmm16 = VPMINSBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMINSDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMINSDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMINSDZ128rr $xmm16, $xmm1 - $xmm16 = VPMINSDZ128rr $xmm16, $xmm1 + $xmm16 = VPMINSDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMINSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMINSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMINSWZ128rr $xmm16, $xmm1 - $xmm16 = VPMINSWZ128rr $xmm16, $xmm1 + $xmm16 = VPMINSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMINUBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMINUBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMINUBZ128rr $xmm16, $xmm1 - $xmm16 = VPMINUBZ128rr $xmm16, $xmm1 + $xmm16 = VPMINUBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMINUDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMINUDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMINUDZ128rr $xmm16, $xmm1 - $xmm16 = VPMINUDZ128rr $xmm16, $xmm1 + $xmm16 = VPMINUDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMINUWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMINUWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMINUWZ128rr $xmm16, $xmm1 - $xmm16 = VPMINUWZ128rr $xmm16, $xmm1 + $xmm16 = VPMINUWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULDQZ128rr $xmm16, $xmm1 - $xmm16 = VPMULDQZ128rr $xmm16, $xmm1 + $xmm16 = VPMULDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULHRSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULHRSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULHRSWZ128rr $xmm16, $xmm1 - $xmm16 = VPMULHRSWZ128rr $xmm16, $xmm1 + $xmm16 = VPMULHRSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULHUWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULHUWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULHUWZ128rr $xmm16, $xmm1 - $xmm16 = VPMULHUWZ128rr $xmm16, $xmm1 + $xmm16 = VPMULHUWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULHWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULHWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULHWZ128rr $xmm16, $xmm1 - $xmm16 = VPMULHWZ128rr $xmm16, $xmm1 + $xmm16 = VPMULHWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULLDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULLDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULLDZ128rr $xmm16, $xmm1 - $xmm16 = VPMULLDZ128rr $xmm16, $xmm1 + $xmm16 = VPMULLDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULLWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULLWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULLWZ128rr $xmm16, $xmm1 - $xmm16 = VPMULLWZ128rr $xmm16, $xmm1 + $xmm16 = VPMULLWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMULUDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMULUDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMULUDQZ128rr $xmm16, $xmm1 - $xmm16 = VPMULUDQZ128rr $xmm16, $xmm1 + $xmm16 = VPMULUDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPORDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPORDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPORDZ128rr $xmm16, $xmm1 - $xmm16 = VPORDZ128rr $xmm16, $xmm1 + $xmm16 = VPORDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPORQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPORQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPORQZ128rr $xmm16, $xmm1 - $xmm16 = VPORQZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPORQZ128rr $xmm16, $xmm1 + $xmm16 = VPORQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSUBBZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBBZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSUBDZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBDZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSUBQZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBQZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPSUBSBZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBSBZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPSUBSBZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBSBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSUBSWZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBSWZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBUSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBUSBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPSUBUSBZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBUSBZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPSUBUSBZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBUSBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBUSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBUSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSUBUSWZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBUSWZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBUSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSUBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSUBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPSUBWZ128rr $xmm16, $xmm1 - $xmm16 = VPSUBWZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPSUBWZ128rr $xmm16, $xmm1 + $xmm16 = VPSUBWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VADDPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VADDPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VADDPDZ128rr $xmm16, $xmm1, implicit $mxcsr @@ -3591,19 +3591,19 @@ body: | ; CHECK: $xmm16 = VANDNPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VANDNPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VANDNPDZ128rr $xmm16, $xmm1 - $xmm16 = VANDNPDZ128rr $xmm16, $xmm1 + $xmm16 = VANDNPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VANDNPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VANDNPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VANDNPSZ128rr $xmm16, $xmm1 - $xmm16 = VANDNPSZ128rr $xmm16, $xmm1 + $xmm16 = VANDNPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VANDPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VANDPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VANDPDZ128rr $xmm16, $xmm1 - $xmm16 = VANDPDZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VANDPDZ128rr $xmm16, $xmm1 + $xmm16 = VANDPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VANDPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VANDPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VANDPSZ128rr $xmm16, $xmm1 - $xmm16 = VANDPSZ128rr $xmm16, $xmm1 + $xmm16 = VANDPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VDIVPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VDIVPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VDIVPDZ128rr $xmm16, $xmm1, implicit $mxcsr @@ -3615,11 +3615,11 @@ body: | ; CHECK: $xmm16 = VPXORDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPXORDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPXORDZ128rr $xmm16, $xmm1 - $xmm16 = VPXORDZ128rr $xmm16, $xmm1 + $xmm16 = VPXORDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPXORQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPXORQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPXORQZ128rr $xmm16, $xmm1 - $xmm16 = VPXORQZ128rr $xmm16, $xmm1 + $xmm16 = VPXORQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VSUBPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VSUBPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VSUBPDZ128rr $xmm16, $xmm1, implicit $mxcsr @@ -3631,83 +3631,83 @@ body: | ; CHECK: $xmm16 = VXORPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VXORPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VXORPDZ128rr $xmm16, $xmm1 - $xmm16 = VXORPDZ128rr $xmm16, $xmm1 + $xmm16 = VXORPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VXORPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VXORPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VXORPSZ128rr $xmm16, $xmm1 - $xmm16 = VXORPSZ128rr $xmm16, $xmm1 + $xmm16 = VXORPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMADDUBSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMADDUBSWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPMADDUBSWZ128rr $xmm16, $xmm1 - $xmm16 = VPMADDUBSWZ128rr $xmm16, $xmm1 + $xmm16 = VPMADDUBSWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPMADDWDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPMADDWDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPMADDWDZ128rr $xmm16, $xmm1 - $xmm16 = VPMADDWDZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPMADDWDZ128rr $xmm16, $xmm1 + $xmm16 = VPMADDWDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPACKSSDWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPACKSSDWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPACKSSDWZ128rr $xmm16, $xmm1 - $xmm16 = VPACKSSDWZ128rr $xmm16, $xmm1 + $xmm16 = VPACKSSDWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPACKSSWBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPACKSSWBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPACKSSWBZ128rr $xmm16, $xmm1 - $xmm16 = VPACKSSWBZ128rr $xmm16, $xmm1 + $xmm16 = VPACKSSWBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPACKUSDWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPACKUSDWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPACKUSDWZ128rr $xmm16, $xmm1 - $xmm16 = VPACKUSDWZ128rr $xmm16, $xmm1 + $xmm16 = VPACKUSDWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPACKUSWBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPACKUSWBZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPACKUSWBZ128rr $xmm16, $xmm1 - $xmm16 = VPACKUSWBZ128rr $xmm16, $xmm1 + $xmm16 = VPACKUSWBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKHBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKHBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKHBWZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKHBWZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKHBWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKHDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKHDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKHDQZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKHDQZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKHDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKHQDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKHQDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKHQDQZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKHQDQZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKHQDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKHWDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKHWDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKHWDZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKHWDZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKHWDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKLBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKLBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKLBWZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKLBWZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKLBWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKLDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKLDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKLDQZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKLDQZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKLDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKLQDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKLQDQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKLQDQZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKLQDQZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKLQDQZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPUNPCKLWDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPUNPCKLWDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPUNPCKLWDZ128rr $xmm16, $xmm1 - $xmm16 = VPUNPCKLWDZ128rr $xmm16, $xmm1 + $xmm16 = VPUNPCKLWDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VUNPCKHPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VUNPCKHPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VUNPCKHPDZ128rr $xmm16, $xmm1 - $xmm16 = VUNPCKHPDZ128rr $xmm16, $xmm1 + $xmm16 = VUNPCKHPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VUNPCKHPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VUNPCKHPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VUNPCKHPSZ128rr $xmm16, $xmm1 - $xmm16 = VUNPCKHPSZ128rr $xmm16, $xmm1 + $xmm16 = VUNPCKHPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VUNPCKLPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VUNPCKLPDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VUNPCKLPDZ128rr $xmm16, $xmm1 - $xmm16 = VUNPCKLPDZ128rr $xmm16, $xmm1 + $xmm16 = VUNPCKLPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VUNPCKLPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VUNPCKLPSZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VUNPCKLPSZ128rr $xmm16, $xmm1 - $xmm16 = VUNPCKLPSZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VUNPCKLPSZ128rr $xmm16, $xmm1 + $xmm16 = VUNPCKLPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VFMADD132PDZ128m $xmm16, $xmm16, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VFMADD132PDZ128m $xmm16, $xmm16, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VFMADD132PDZ128r $xmm16, $xmm1, $xmm2, implicit $mxcsr @@ -3852,14 +3852,14 @@ body: | $xmm16 = VFNMSUB231PSZ128m $xmm16, $xmm16, $rsi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VFNMSUB231PSZ128r $xmm16, $xmm1, $xmm2, implicit $mxcsr $xmm16 = VFNMSUB231PSZ128r $xmm16, $xmm1, $xmm2, implicit $mxcsr - ; CHECK: $xmm16 = VPSLLDZ128ri $xmm16, 7 - $xmm16 = VPSLLDZ128ri $xmm16, 7 + ; CHECK: $xmm16 = VPSLLDZ128ri $xmm16, 7 + $xmm16 = VPSLLDZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSLLDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSLLDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSLLDZ128rr $xmm16, $xmm16 $xmm16 = VPSLLDZ128rr $xmm16, $xmm16 ; CHECK: $xmm16 = VPSLLQZ128ri $xmm16, 7 - $xmm16 = VPSLLQZ128ri $xmm16, 7 + $xmm16 = VPSLLQZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSLLQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSLLQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSLLQZ128rr $xmm16, $xmm16 @@ -3873,13 +3873,13 @@ body: | ; CHECK: $xmm16 = VPSLLVQZ128rr $xmm16, $xmm16 $xmm16 = VPSLLVQZ128rr $xmm16, $xmm16 ; CHECK: $xmm16 = VPSLLWZ128ri $xmm16, 7 - $xmm16 = VPSLLWZ128ri $xmm16, 7 + $xmm16 = VPSLLWZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSLLWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSLLWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSLLWZ128rr $xmm16, $xmm16 $xmm16 = VPSLLWZ128rr $xmm16, $xmm16 ; CHECK: $xmm16 = VPSRADZ128ri $xmm16, 7 - $xmm16 = VPSRADZ128ri $xmm16, 7 + $xmm16 = VPSRADZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSRADZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSRADZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSRADZ128rr $xmm16, $xmm16 @@ -3888,22 +3888,22 @@ body: | $xmm16 = VPSRAVDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSRAVDZ128rr $xmm16, $xmm16 $xmm16 = VPSRAVDZ128rr $xmm16, $xmm16 - ; CHECK: $xmm16 = VPSRAWZ128ri $xmm16, 7 - $xmm16 = VPSRAWZ128ri $xmm16, 7 + ; CHECK: $xmm16 = VPSRAWZ128ri $xmm16, 7 + $xmm16 = VPSRAWZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSRAWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSRAWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSRAWZ128rr $xmm16, $xmm16 $xmm16 = VPSRAWZ128rr $xmm16, $xmm16 ; CHECK: $xmm16 = VPSRLDQZ128ri $xmm16, 14 - $xmm16 = VPSRLDQZ128ri $xmm16, 14 - ; CHECK: $xmm16 = VPSRLDZ128ri $xmm16, 7 - $xmm16 = VPSRLDZ128ri $xmm16, 7 + $xmm16 = VPSRLDQZ128ri $xmm16, 14 + ; CHECK: $xmm16 = VPSRLDZ128ri $xmm16, 7 + $xmm16 = VPSRLDZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSRLDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSRLDZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSRLDZ128rr $xmm16, $xmm16 $xmm16 = VPSRLDZ128rr $xmm16, $xmm16 - ; CHECK: $xmm16 = VPSRLQZ128ri $xmm16, 7 - $xmm16 = VPSRLQZ128ri $xmm16, 7 + ; CHECK: $xmm16 = VPSRLQZ128ri $xmm16, 7 + $xmm16 = VPSRLQZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSRLQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSRLQZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSRLQZ128rr $xmm16, $xmm16 @@ -3917,7 +3917,7 @@ body: | ; CHECK: $xmm16 = VPSRLVQZ128rr $xmm16, $xmm16 $xmm16 = VPSRLVQZ128rr $xmm16, $xmm16 ; CHECK: $xmm16 = VPSRLWZ128ri $xmm16, 7 - $xmm16 = VPSRLWZ128ri $xmm16, 7 + $xmm16 = VPSRLWZ128ri $xmm16, 7 ; CHECK: $xmm16 = VPSRLWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSRLWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSRLWZ128rr $xmm16, $xmm16 @@ -3925,27 +3925,27 @@ body: | ; CHECK: $xmm16 = VPERMILPDZ128mi $rdi, 1, $noreg, 0, $noreg, 9 $xmm16 = VPERMILPDZ128mi $rdi, 1, $noreg, 0, $noreg, 9 ; CHECK: $xmm16 = VPERMILPDZ128ri $xmm16, 9 - $xmm16 = VPERMILPDZ128ri $xmm16, 9 + $xmm16 = VPERMILPDZ128ri $xmm16, 9 ; CHECK: $xmm16 = VPERMILPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VPERMILPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VPERMILPDZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPERMILPDZ128rr $xmm16, $xmm1 - $xmm16 = VPERMILPDZ128rr $xmm16, $xmm1 + $xmm16 = VPERMILPDZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPERMILPSZ128mi $rdi, 1, $noreg, 0, $noreg, 9 $xmm16 = VPERMILPSZ128mi $rdi, 1, $noreg, 0, $noreg, 9 ; CHECK: $xmm16 = VPERMILPSZ128ri $xmm16, 9 - $xmm16 = VPERMILPSZ128ri $xmm16, 9 + $xmm16 = VPERMILPSZ128ri $xmm16, 9 ; CHECK: $xmm16 = VPERMILPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VPERMILPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VPERMILPSZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPERMILPSZ128rr $xmm16, $xmm1 - $xmm16 = VPERMILPSZ128rr $xmm16, $xmm1 + $xmm16 = VPERMILPSZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VCVTPH2PSZ128rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VCVTPH2PSZ128rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VCVTPH2PSZ128rr $xmm16, implicit $mxcsr $xmm16 = VCVTPH2PSZ128rr $xmm16, implicit $mxcsr ; CHECK: $xmm16 = VCVTDQ2PDZ128rm $rdi, 1, $noreg, 0, $noreg $xmm16 = VCVTDQ2PDZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VCVTDQ2PDZ128rr $xmm16 - $xmm16 = VCVTDQ2PDZ128rr $xmm16 + ; CHECK: $xmm16 = VCVTDQ2PDZ128rr $xmm16 + $xmm16 = VCVTDQ2PDZ128rr $xmm16 ; CHECK: $xmm16 = VCVTDQ2PSZ128rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VCVTDQ2PSZ128rm $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VCVTDQ2PSZ128rr $xmm16, implicit $mxcsr @@ -3982,34 +3982,34 @@ body: | $xmm16 = VSQRTPSZ128m $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VSQRTPSZ128r $xmm16, implicit $mxcsr $xmm16 = VSQRTPSZ128r $xmm16, implicit $mxcsr - ; CHECK: $xmm16 = VMOVDDUPZ128rm $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVDDUPZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VMOVDDUPZ128rr $xmm16 - $xmm16 = VMOVDDUPZ128rr $xmm16 - ; CHECK: $xmm16 = VMOVSHDUPZ128rm $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVSHDUPZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VMOVSHDUPZ128rr $xmm16 - $xmm16 = VMOVSHDUPZ128rr $xmm16 - ; CHECK: $xmm16 = VMOVSLDUPZ128rm $rdi, 1, $noreg, 0, $noreg - $xmm16 = VMOVSLDUPZ128rm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VMOVSLDUPZ128rr $xmm16 - $xmm16 = VMOVSLDUPZ128rr $xmm16 + ; CHECK: $xmm16 = VMOVDDUPZ128rm $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVDDUPZ128rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm16 = VMOVDDUPZ128rr $xmm16 + $xmm16 = VMOVDDUPZ128rr $xmm16 + ; CHECK: $xmm16 = VMOVSHDUPZ128rm $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVSHDUPZ128rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm16 = VMOVSHDUPZ128rr $xmm16 + $xmm16 = VMOVSHDUPZ128rr $xmm16 + ; CHECK: $xmm16 = VMOVSLDUPZ128rm $rdi, 1, $noreg, 0, $noreg + $xmm16 = VMOVSLDUPZ128rm $rdi, 1, $noreg, 0, $noreg + ; CHECK: $xmm16 = VMOVSLDUPZ128rr $xmm16 + $xmm16 = VMOVSLDUPZ128rr $xmm16 ; CHECK: $xmm16 = VPSHUFBZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg $xmm16 = VPSHUFBZ128rm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPSHUFBZ128rr $xmm16, $xmm1 - $xmm16 = VPSHUFBZ128rr $xmm16, $xmm1 + $xmm16 = VPSHUFBZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VPSHUFDZ128mi $rdi, 1, $noreg, 0, $noreg, -24 $xmm16 = VPSHUFDZ128mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $xmm16 = VPSHUFDZ128ri $xmm16, -24 - $xmm16 = VPSHUFDZ128ri $xmm16, -24 + $xmm16 = VPSHUFDZ128ri $xmm16, -24 ; CHECK: $xmm16 = VPSHUFHWZ128mi $rdi, 1, $noreg, 0, $noreg, -24 $xmm16 = VPSHUFHWZ128mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $xmm16 = VPSHUFHWZ128ri $xmm16, -24 - $xmm16 = VPSHUFHWZ128ri $xmm16, -24 + $xmm16 = VPSHUFHWZ128ri $xmm16, -24 ; CHECK: $xmm16 = VPSHUFLWZ128mi $rdi, 1, $noreg, 0, $noreg, -24 $xmm16 = VPSHUFLWZ128mi $rdi, 1, $noreg, 0, $noreg, -24 ; CHECK: $xmm16 = VPSHUFLWZ128ri $xmm16, -24 - $xmm16 = VPSHUFLWZ128ri $xmm16, -24 + $xmm16 = VPSHUFLWZ128ri $xmm16, -24 ; CHECK: $xmm16 = VPSLLDQZ128ri $xmm16, 1 $xmm16 = VPSLLDQZ128ri $xmm16, 1 ; CHECK: $xmm16 = VSHUFPDZ128rmi $xmm16, $rip, 1, $noreg, 0, $noreg, -24 @@ -4022,28 +4022,28 @@ body: | $xmm16 = VSHUFPSZ128rri $xmm16, $xmm1, -24 ; CHECK: $xmm16 = VPSADBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg $xmm16 = VPSADBWZ128rm $xmm16, $rip, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VPSADBWZ128rr $xmm16, $xmm1 - $xmm16 = VPSADBWZ128rr $xmm16, $xmm1 + ; CHECK: $xmm16 = VPSADBWZ128rr $xmm16, $xmm1 + $xmm16 = VPSADBWZ128rr $xmm16, $xmm1 ; CHECK: $xmm16 = VBROADCASTSSZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VBROADCASTSSZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VBROADCASTSSZ128rr $xmm16 - $xmm16 = VBROADCASTSSZ128rr $xmm16 + $xmm16 = VBROADCASTSSZ128rr $xmm16 ; CHECK: $xmm16 = VPBROADCASTBZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPBROADCASTBZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPBROADCASTBZ128rr $xmm16 - $xmm16 = VPBROADCASTBZ128rr $xmm16 + $xmm16 = VPBROADCASTBZ128rr $xmm16 ; CHECK: $xmm16 = VPBROADCASTDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPBROADCASTDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPBROADCASTDZ128rr $xmm16 - $xmm16 = VPBROADCASTDZ128rr $xmm16 + $xmm16 = VPBROADCASTDZ128rr $xmm16 ; CHECK: $xmm16 = VPBROADCASTQZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPBROADCASTQZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPBROADCASTQZ128rr $xmm16 - $xmm16 = VPBROADCASTQZ128rr $xmm16 + $xmm16 = VPBROADCASTQZ128rr $xmm16 ; CHECK: $xmm16 = VPBROADCASTWZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPBROADCASTWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPBROADCASTWZ128rr $xmm16 - $xmm16 = VPBROADCASTWZ128rr $xmm16 + $xmm16 = VPBROADCASTWZ128rr $xmm16 ; CHECK: $xmm16 = VBROADCASTI32X2Z128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VBROADCASTI32X2Z128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VBROADCASTI32X2Z128rr $xmm0 @@ -4055,15 +4055,15 @@ body: | ; CHECK: $xmm16 = VPABSBZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPABSBZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPABSBZ128rr $xmm16 - $xmm16 = VPABSBZ128rr $xmm16 + $xmm16 = VPABSBZ128rr $xmm16 ; CHECK: $xmm16 = VPABSDZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPABSDZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPABSDZ128rr $xmm16 - $xmm16 = VPABSDZ128rr $xmm16 + $xmm16 = VPABSDZ128rr $xmm16 ; CHECK: $xmm16 = VPABSWZ128rm $rip, 1, $noreg, 0, $noreg $xmm16 = VPABSWZ128rm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VPABSWZ128rr $xmm16 - $xmm16 = VPABSWZ128rr $xmm16 + $xmm16 = VPABSWZ128rr $xmm16 ; CHECK: $xmm16 = VPALIGNRZ128rmi $xmm16, $rdi, 1, $noreg, 0, $noreg, 15 $xmm16 = VPALIGNRZ128rmi $xmm16, $rdi, 1, $noreg, 0, $noreg, 15 ; CHECK: $xmm16 = VPALIGNRZ128rri $xmm16, $xmm1, 15 @@ -4092,12 +4092,12 @@ body: | $xmm0 = VRNDSCALEPSZ128rmi $rip, 1, $noreg, 0, $noreg, 31, implicit $mxcsr ; CHECK: $xmm0 = VRNDSCALEPSZ128rri $xmm0, 31, implicit $mxcsr $xmm0 = VRNDSCALEPSZ128rri $xmm0, 31, implicit $mxcsr - + RET64 ... --- # CHECK-LABEL: name: evex_scalar_to_evex_test - # CHECK: bb.0: + # CHECK: bb.0: name: evex_scalar_to_evex_test body: | @@ -4406,38 +4406,38 @@ body: | $xmm16 = VFNMSUB231SSZr $xmm16, $xmm1, $xmm2, implicit $mxcsr ; CHECK: $xmm16 = VFNMSUB231SSZr_Int $xmm16, $xmm1, $xmm2, implicit $mxcsr $xmm16 = VFNMSUB231SSZr_Int $xmm16, $xmm1, $xmm2, implicit $mxcsr - ; CHECK: VPEXTRBZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - VPEXTRBZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - ; CHECK: $eax = VPEXTRBZrr $xmm16, 1 - $eax = VPEXTRBZrr $xmm16, 1 - ; CHECK: VPEXTRDZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - VPEXTRDZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - ; CHECK: $eax = VPEXTRDZrr $xmm16, 1 - $eax = VPEXTRDZrr $xmm16, 1 - ; CHECK: VPEXTRQZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - VPEXTRQZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - ; CHECK: $rax = VPEXTRQZrr $xmm16, 1 - $rax = VPEXTRQZrr $xmm16, 1 - ; CHECK: VPEXTRWZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - VPEXTRWZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 - ; CHECK: $eax = VPEXTRWZrr $xmm16, 1 - $eax = VPEXTRWZrr $xmm16, 1 - ; CHECK: $xmm16 = VPINSRBZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm16 = VPINSRBZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm16 = VPINSRBZrr $xmm16, $edi, 5 - $xmm16 = VPINSRBZrr $xmm16, $edi, 5 - ; CHECK: $xmm16 = VPINSRDZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm16 = VPINSRDZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm16 = VPINSRDZrr $xmm16, $edi, 5 - $xmm16 = VPINSRDZrr $xmm16, $edi, 5 - ; CHECK: $xmm16 = VPINSRQZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm16 = VPINSRQZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - ; CHECK: $xmm16 = VPINSRQZrr $xmm16, $rdi, 5 - $xmm16 = VPINSRQZrr $xmm16, $rdi, 5 - ; CHECK: $xmm16 = VPINSRWZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 - $xmm16 = VPINSRWZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: VPEXTRBZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + VPEXTRBZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + ; CHECK: $eax = VPEXTRBZrr $xmm16, 1 + $eax = VPEXTRBZrr $xmm16, 1 + ; CHECK: VPEXTRDZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + VPEXTRDZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + ; CHECK: $eax = VPEXTRDZrr $xmm16, 1 + $eax = VPEXTRDZrr $xmm16, 1 + ; CHECK: VPEXTRQZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + VPEXTRQZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + ; CHECK: $rax = VPEXTRQZrr $xmm16, 1 + $rax = VPEXTRQZrr $xmm16, 1 + ; CHECK: VPEXTRWZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + VPEXTRWZmr $rdi, 1, $noreg, 0, $noreg, $xmm16, 3 + ; CHECK: $eax = VPEXTRWZrr $xmm16, 1 + $eax = VPEXTRWZrr $xmm16, 1 + ; CHECK: $xmm16 = VPINSRBZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm16 = VPINSRBZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm16 = VPINSRBZrr $xmm16, $edi, 5 + $xmm16 = VPINSRBZrr $xmm16, $edi, 5 + ; CHECK: $xmm16 = VPINSRDZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm16 = VPINSRDZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm16 = VPINSRDZrr $xmm16, $edi, 5 + $xmm16 = VPINSRDZrr $xmm16, $edi, 5 + ; CHECK: $xmm16 = VPINSRQZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm16 = VPINSRQZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + ; CHECK: $xmm16 = VPINSRQZrr $xmm16, $rdi, 5 + $xmm16 = VPINSRQZrr $xmm16, $rdi, 5 + ; CHECK: $xmm16 = VPINSRWZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 + $xmm16 = VPINSRWZrm $xmm16, $rsi, 1, $noreg, 0, $noreg, 3 ; CHECK: $xmm16 = VPINSRWZrr $xmm16, $edi, 5 - $xmm16 = VPINSRWZrr $xmm16, $edi, 5 + $xmm16 = VPINSRWZrr $xmm16, $edi, 5 ; CHECK: $xmm16 = VSQRTSDZm $xmm16, $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr $xmm16 = VSQRTSDZm $xmm16, $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr ; CHECK: $xmm16 = VSQRTSDZm_Int $xmm16, $rdi, 1, $noreg, 0, $noreg, implicit $mxcsr @@ -4471,9 +4471,9 @@ body: | ; CHECK: $xmm16 = VCVTSD2SSZrr_Int $xmm16, $xmm16, implicit $mxcsr $xmm16 = VCVTSD2SSZrr_Int $xmm16, $xmm16, implicit $mxcsr ; CHECK: $xmm16 = VCVTSI2SDZrm $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VCVTSI2SDZrm $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VCVTSI2SDZrm $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VCVTSI2SDZrm_Int $xmm16, $rdi, 1, $noreg, 0, $noreg - $xmm16 = VCVTSI2SDZrm_Int $xmm16, $rdi, 1, $noreg, 0, $noreg + $xmm16 = VCVTSI2SDZrm_Int $xmm16, $rdi, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VCVTSI2SDZrr $xmm16, $edi $xmm16 = VCVTSI2SDZrr $xmm16, $edi ; CHECK: $xmm16 = VCVTSI2SDZrr_Int $xmm16, $edi @@ -4550,10 +4550,10 @@ body: | $edi = VCVTTSS2SIZrr $xmm16, implicit $mxcsr ; CHECK: $edi = VCVTTSS2SIZrr_Int $xmm16, implicit $mxcsr $edi = VCVTTSS2SIZrr_Int $xmm16, implicit $mxcsr - ; CHECK: $xmm16 = VMOV64toSDZrr $rdi - $xmm16 = VMOV64toSDZrr $rdi + ; CHECK: $xmm16 = VMOV64toSDZrr $rdi + $xmm16 = VMOV64toSDZrr $rdi ; CHECK: $xmm16 = VMOVDI2SSZrr $eax - $xmm16 = VMOVDI2SSZrr $eax + $xmm16 = VMOVDI2SSZrr $eax ; CHECK: VMOVSDZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 VMOVSDZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVSDZrm $rip, 1, $noreg, 0, $noreg @@ -4561,7 +4561,7 @@ body: | ; CHECK: $xmm16 = VMOVSDZrm_alt $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVSDZrm_alt $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVSDZrr $xmm16, $xmm1 - $xmm16 = VMOVSDZrr $xmm16, $xmm1 + $xmm16 = VMOVSDZrr $xmm16, $xmm1 ; CHECK: $rax = VMOVSDto64Zrr $xmm16 $rax = VMOVSDto64Zrr $xmm16 ; CHECK: VMOVSSZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 @@ -4571,19 +4571,19 @@ body: | ; CHECK: $xmm16 = VMOVSSZrm_alt $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVSSZrm_alt $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVSSZrr $xmm16, $xmm1 - $xmm16 = VMOVSSZrr $xmm16, $xmm1 + $xmm16 = VMOVSSZrr $xmm16, $xmm1 ; CHECK: $eax = VMOVSS2DIZrr $xmm16 $eax = VMOVSS2DIZrr $xmm16 ; CHECK: $xmm16 = VMOV64toPQIZrr $rdi - $xmm16 = VMOV64toPQIZrr $rdi + $xmm16 = VMOV64toPQIZrr $rdi ; CHECK: $xmm16 = VMOV64toPQIZrm $rdi, 1, $noreg, 0, $noreg $xmm16 = VMOV64toPQIZrm $rdi, 1, $noreg, 0, $noreg - ; CHECK: $xmm16 = VMOV64toSDZrr $rdi - $xmm16 = VMOV64toSDZrr $rdi + ; CHECK: $xmm16 = VMOV64toSDZrr $rdi + $xmm16 = VMOV64toSDZrr $rdi ; CHECK: $xmm16 = VMOVDI2PDIZrm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVDI2PDIZrm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVDI2PDIZrr $edi - $xmm16 = VMOVDI2PDIZrr $edi + $xmm16 = VMOVDI2PDIZrr $edi ; CHECK: $xmm16 = VMOVLHPSZrr $xmm16, $xmm1 $xmm16 = VMOVLHPSZrr $xmm16, $xmm1 ; CHECK: $xmm16 = VMOVHLPSZrr $xmm16, $xmm1 @@ -4591,19 +4591,19 @@ body: | ; CHECK: VMOVPDI2DIZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 VMOVPDI2DIZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $edi = VMOVPDI2DIZrr $xmm16 - $edi = VMOVPDI2DIZrr $xmm16 + $edi = VMOVPDI2DIZrr $xmm16 ; CHECK: $xmm16 = VMOVPQI2QIZrr $xmm16 $xmm16 = VMOVPQI2QIZrr $xmm16 ; CHECK: VMOVPQI2QIZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 VMOVPQI2QIZmr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $rdi = VMOVPQIto64Zrr $xmm16 - $rdi = VMOVPQIto64Zrr $xmm16 + $rdi = VMOVPQIto64Zrr $xmm16 ; CHECK: VMOVPQIto64Zmr $rdi, 1, $noreg, 0, $noreg, $xmm16 VMOVPQIto64Zmr $rdi, 1, $noreg, 0, $noreg, $xmm16 ; CHECK: $xmm16 = VMOVQI2PQIZrm $rip, 1, $noreg, 0, $noreg $xmm16 = VMOVQI2PQIZrm $rip, 1, $noreg, 0, $noreg ; CHECK: $xmm16 = VMOVZPQILo2PQIZrr $xmm16 - $xmm16 = VMOVZPQILo2PQIZrr $xmm16 + $xmm16 = VMOVZPQILo2PQIZrr $xmm16 ; CHECK: VCOMISDZrm_Int $xmm16, $rdi, 1, $noreg, 0, $noreg, implicit-def $eflags, implicit $mxcsr VCOMISDZrm_Int $xmm16, $rdi, 1, $noreg, 0, $noreg, implicit-def $eflags, implicit $mxcsr ; CHECK: VCOMISDZrr_Int $xmm16, $xmm1, implicit-def $eflags, implicit $mxcsr @@ -4668,6 +4668,6 @@ body: | $xmm0 = VRNDSCALESSZm_Int $xmm0, $rip, 1, $noreg, 0, $noreg, 31, implicit $mxcsr ; CHECK: $xmm0 = VRNDSCALESSZr_Int $xmm0, $xmm1, 31, implicit $mxcsr $xmm0 = VRNDSCALESSZr_Int $xmm0, $xmm1, 31, implicit $mxcsr - + RET64 ... diff --git a/llvm/test/CodeGen/X86/fold-load-vec.ll b/llvm/test/CodeGen/X86/fold-load-vec.ll index 348929cdf9f7..0bf846a0930b 100644 --- a/llvm/test/CodeGen/X86/fold-load-vec.ll +++ b/llvm/test/CodeGen/X86/fold-load-vec.ll @@ -10,8 +10,8 @@ define void @sample_test(ptr %source, ptr %dest) nounwind { ; CHECK-NEXT: subq $24, %rsp ; CHECK-NEXT: movq %rdi, {{[0-9]+}}(%rsp) ; CHECK-NEXT: movq %rsi, {{[0-9]+}}(%rsp) +; CHECK-NEXT: movq $0, (%rsp) ; CHECK-NEXT: xorps %xmm0, %xmm0 -; CHECK-NEXT: movlps %xmm0, (%rsp) ; CHECK-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],mem[0],xmm0[1],mem[1] ; CHECK-NEXT: movlps %xmm0, (%rsp) ; CHECK-NEXT: movlps %xmm0, (%rsi) diff --git a/llvm/test/CodeGen/X86/fold-pcmpeqd-2.ll b/llvm/test/CodeGen/X86/fold-pcmpeqd-2.ll index 5dcb1d63207d..493e7aa7039e 100644 --- a/llvm/test/CodeGen/X86/fold-pcmpeqd-2.ll +++ b/llvm/test/CodeGen/X86/fold-pcmpeqd-2.ll @@ -51,11 +51,6 @@ define void @program_1(ptr %dest, ptr %t0, <4 x float> %p0, <4 x float> %p1, <4 ; X86-NEXT: movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload ; X86-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 ; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill -; X86-NEXT: xorps %xmm0, %xmm0 -; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill -; X86-NEXT: movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload -; X86-NEXT: mulps %xmm0, %xmm0 -; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill ; X86-NEXT: movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload ; X86-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 ; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill @@ -64,8 +59,10 @@ define void @program_1(ptr %dest, ptr %t0, <4 x float> %p0, <4 x float> %p1, <4 ; X86-NEXT: movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload ; X86-NEXT: cmpunordps %xmm0, %xmm0 ; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill +; X86-NEXT: xorps %xmm0, %xmm0 +; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill ; X86-NEXT: movaps {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 ## 16-byte Reload -; X86-NEXT: minps {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0 +; X86-NEXT: minps %xmm0, %xmm0 ; X86-NEXT: movaps %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) ## 16-byte Spill ; X86-NEXT: xorps %xmm0, %xmm0 ; X86-NEXT: movaps %xmm0, {{[0-9]+}}(%esp) @@ -135,11 +132,6 @@ define void @program_1(ptr %dest, ptr %t0, <4 x float> %p0, <4 x float> %p1, <4 ; X64-NEXT: movaps (%rsp), %xmm0 ## 16-byte Reload ; X64-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; X64-NEXT: movaps %xmm0, (%rsp) ## 16-byte Spill -; X64-NEXT: xorps %xmm0, %xmm0 -; X64-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill -; X64-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload -; X64-NEXT: mulps %xmm0, %xmm0 -; X64-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill ; X64-NEXT: movaps (%rsp), %xmm0 ## 16-byte Reload ; X64-NEXT: mulps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; X64-NEXT: movaps %xmm0, (%rsp) ## 16-byte Spill @@ -148,8 +140,10 @@ define void @program_1(ptr %dest, ptr %t0, <4 x float> %p0, <4 x float> %p1, <4 ; X64-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload ; X64-NEXT: cmpunordps %xmm0, %xmm0 ; X64-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill +; X64-NEXT: xorps %xmm0, %xmm0 +; X64-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill ; X64-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 ## 16-byte Reload -; X64-NEXT: minps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: minps %xmm0, %xmm0 ; X64-NEXT: movaps %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) ## 16-byte Spill ; X64-NEXT: xorl %ebx, %ebx ; X64-NEXT: xorps %xmm3, %xmm3 diff --git a/llvm/test/CodeGen/X86/fp-intrinsics.ll b/llvm/test/CodeGen/X86/fp-intrinsics.ll index 32e45adcb94d..5f77e2cb46cb 100644 --- a/llvm/test/CodeGen/X86/fp-intrinsics.ll +++ b/llvm/test/CodeGen/X86/fp-intrinsics.ll @@ -865,9 +865,9 @@ define double @f19() #0 { ; X87-NEXT: .cfi_def_cfa_offset 32 ; X87-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}} ; X87-NEXT: fstpl {{[0-9]+}}(%esp) +; X87-NEXT: fld1 +; X87-NEXT: fstpl (%esp) ; X87-NEXT: wait -; X87-NEXT: movl $1072693248, {{[0-9]+}}(%esp) # imm = 0x3FF00000 -; X87-NEXT: movl $0, (%esp) ; X87-NEXT: calll fmod ; X87-NEXT: addl $28, %esp ; X87-NEXT: .cfi_def_cfa_offset 4 diff --git a/llvm/test/CodeGen/X86/gep-expanded-vector.ll b/llvm/test/CodeGen/X86/gep-expanded-vector.ll index 861de9c90085..943cd3610c9d 100644 --- a/llvm/test/CodeGen/X86/gep-expanded-vector.ll +++ b/llvm/test/CodeGen/X86/gep-expanded-vector.ll @@ -1,24 +1,42 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 ; RUN: llc < %s -O2 -mattr=avx512f -mtriple=x86_64-unknown | FileCheck %s %struct.S1 = type { ptr, ptr } -define ptr @malloc_init_state(<64 x ptr> %tmp, i32 %ind) { +define ptr @malloc_init_state(<64 x ptr> %tmp, i32 %ind) nounwind { +; CHECK-LABEL: malloc_init_state: +; CHECK: # %bb.0: # %entry +; CHECK-NEXT: pushq %rbp +; CHECK-NEXT: movq %rsp, %rbp +; CHECK-NEXT: andq $-64, %rsp +; CHECK-NEXT: subq $576, %rsp # imm = 0x240 +; CHECK-NEXT: # kill: def $edi killed $edi def $rdi +; CHECK-NEXT: vpbroadcastq {{.*#+}} zmm8 = [16,16,16,16,16,16,16,16] +; CHECK-NEXT: vpaddq %zmm8, %zmm0, %zmm0 +; CHECK-NEXT: vpaddq %zmm8, %zmm1, %zmm1 +; CHECK-NEXT: vpaddq %zmm8, %zmm2, %zmm2 +; CHECK-NEXT: vpaddq %zmm8, %zmm3, %zmm3 +; CHECK-NEXT: vpaddq %zmm8, %zmm4, %zmm4 +; CHECK-NEXT: vpaddq %zmm8, %zmm5, %zmm5 +; CHECK-NEXT: vpaddq %zmm8, %zmm6, %zmm6 +; CHECK-NEXT: vpaddq %zmm8, %zmm7, %zmm7 +; CHECK-NEXT: vmovdqa64 %zmm7, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm6, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm5, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm4, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm3, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm2, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp) +; CHECK-NEXT: vmovdqa64 %zmm0, (%rsp) +; CHECK-NEXT: andl $63, %edi +; CHECK-NEXT: movq (%rsp,%rdi,8), %rax +; CHECK-NEXT: movq %rbp, %rsp +; CHECK-NEXT: popq %rbp +; CHECK-NEXT: vzeroupper +; CHECK-NEXT: retq entry: %Vec = getelementptr inbounds ptr, <64 x ptr> %tmp , i64 2 %ptr = extractelement <64 x ptr> %Vec, i32 %ind ret ptr %ptr } -; CHECK: .LCPI0_0: -; CHECK: .quad 16 -; CHECK: vpbroadcastq .LCPI0_0(%rip), [[Z1:%zmm[0-9]]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z2:%zmm[0-9]]], [[Z2]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z3:%zmm[0-9]]], [[Z3]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z4:%zmm[0-9]]], [[Z4]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z5:%zmm[0-9]]], [[Z5]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z6:%zmm[0-9]]], [[Z6]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z7:%zmm[0-9]]], [[Z7]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z8:%zmm[0-9]]], [[Z8]] -; CHECK-NEXT: vpaddq [[Z1]], [[Z9:%zmm[0-9]]], [[Z9]] - - diff --git a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll index 6cd85e074c64..8f1272013c11 100644 --- a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll +++ b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll @@ -31,7 +31,7 @@ define <16 x i8> @splatconstant_fshl_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $3, %xmm0, %xmm2 ; GFNIAVX512-NEXT: vpsrlw $5, %xmm1, %xmm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; GFNIAVX512-NEXT: retq %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %b, <16 x i8> ) ret <16 x i8> %res @@ -119,7 +119,7 @@ define <32 x i8> @splatconstant_fshl_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $4, %ymm0, %ymm2 ; GFNIAVX512-NEXT: vpsrlw $4, %ymm1, %ymm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; GFNIAVX512-NEXT: retq %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> ) ret <32 x i8> %res @@ -175,7 +175,7 @@ define <32 x i8> @splatconstant_fshr_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $2, %ymm0, %ymm2 ; GFNIAVX512-NEXT: vpsrlw $6, %ymm1, %ymm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; GFNIAVX512-NEXT: retq %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a, <32 x i8> %b, <32 x i8> ) ret <32 x i8> %res @@ -339,7 +339,7 @@ define <64 x i8> @splatconstant_fshr_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $6, %zmm0, %zmm2 ; GFNIAVX512-NEXT: vpsrlw $2, %zmm1, %zmm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; GFNIAVX512-NEXT: retq %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> ) ret <64 x i8> %res diff --git a/llvm/test/CodeGen/X86/gfni-rotates.ll b/llvm/test/CodeGen/X86/gfni-rotates.ll index 29b58d047596..8b94a3f84cb9 100644 --- a/llvm/test/CodeGen/X86/gfni-rotates.ll +++ b/llvm/test/CodeGen/X86/gfni-rotates.ll @@ -32,7 +32,7 @@ define <16 x i8> @splatconstant_rotl_v16i8(<16 x i8> %a) nounwind { ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $3, %xmm0, %xmm1 ; GFNIAVX512-NEXT: vpsrlw $5, %xmm0, %xmm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; GFNIAVX512-NEXT: retq %res = call <16 x i8> @llvm.fshl.v16i8(<16 x i8> %a, <16 x i8> %a, <16 x i8> ) ret <16 x i8> %res @@ -121,7 +121,7 @@ define <32 x i8> @splatconstant_rotl_v32i8(<32 x i8> %a) nounwind { ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $4, %ymm0, %ymm1 ; GFNIAVX512-NEXT: vpsrlw $4, %ymm0, %ymm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; GFNIAVX512-NEXT: retq %res = call <32 x i8> @llvm.fshl.v32i8(<32 x i8> %a, <32 x i8> %a, <32 x i8> ) ret <32 x i8> %res @@ -177,7 +177,7 @@ define <32 x i8> @splatconstant_rotr_v32i8(<32 x i8> %a) nounwind { ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $2, %ymm0, %ymm1 ; GFNIAVX512-NEXT: vpsrlw $6, %ymm0, %ymm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; GFNIAVX512-NEXT: retq %res = call <32 x i8> @llvm.fshr.v32i8(<32 x i8> %a, <32 x i8> %a, <32 x i8> ) ret <32 x i8> %res @@ -344,7 +344,7 @@ define <64 x i8> @splatconstant_rotr_v64i8(<64 x i8> %a) nounwind { ; GFNIAVX512: # %bb.0: ; GFNIAVX512-NEXT: vpsllw $6, %zmm0, %zmm1 ; GFNIAVX512-NEXT: vpsrlw $2, %zmm0, %zmm0 -; GFNIAVX512-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; GFNIAVX512-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; GFNIAVX512-NEXT: retq %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %a, <64 x i8> %a, <64 x i8> ) ret <64 x i8> %res diff --git a/llvm/test/CodeGen/X86/half.ll b/llvm/test/CodeGen/X86/half.ll index 596e465ee8ca..722525720316 100644 --- a/llvm/test/CodeGen/X86/half.ll +++ b/llvm/test/CodeGen/X86/half.ll @@ -1082,12 +1082,11 @@ define void @main.158() #0 { ; BWON-F16C-LABEL: main.158: ; BWON-F16C: # %bb.0: # %entry ; BWON-F16C-NEXT: vxorps %xmm0, %xmm0, %xmm0 -; BWON-F16C-NEXT: vcvtps2ph $4, %xmm0, %xmm0 -; BWON-F16C-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; BWON-F16C-NEXT: vcvtph2ps %xmm0, %xmm0 -; BWON-F16C-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero -; BWON-F16C-NEXT: vucomiss %xmm0, %xmm1 -; BWON-F16C-NEXT: vxorps %xmm0, %xmm0, %xmm0 +; BWON-F16C-NEXT: vcvtps2ph $4, %xmm0, %xmm1 +; BWON-F16C-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero +; BWON-F16C-NEXT: vcvtph2ps %xmm1, %xmm1 +; BWON-F16C-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero +; BWON-F16C-NEXT: vucomiss %xmm1, %xmm2 ; BWON-F16C-NEXT: jae .LBB20_2 ; BWON-F16C-NEXT: # %bb.1: # %entry ; BWON-F16C-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero @@ -1100,8 +1099,7 @@ define void @main.158() #0 { ; CHECK-I686-LABEL: main.158: ; CHECK-I686: # %bb.0: # %entry ; CHECK-I686-NEXT: subl $12, %esp -; CHECK-I686-NEXT: pxor %xmm0, %xmm0 -; CHECK-I686-NEXT: movd %xmm0, (%esp) +; CHECK-I686-NEXT: movl $0, (%esp) ; CHECK-I686-NEXT: calll __truncsfhf2 ; CHECK-I686-NEXT: pextrw $0, %xmm0, %eax ; CHECK-I686-NEXT: movw %ax, (%esp) diff --git a/llvm/test/CodeGen/X86/ldexp.ll b/llvm/test/CodeGen/X86/ldexp.ll index 44c57c54ba02..ec128fc6686c 100644 --- a/llvm/test/CodeGen/X86/ldexp.ll +++ b/llvm/test/CodeGen/X86/ldexp.ll @@ -91,10 +91,11 @@ define double @ldexp_f64(i8 zeroext %x) { ; ; WIN32-LABEL: ldexp_f64: ; WIN32: # %bb.0: +; WIN32-NEXT: subl $12, %esp ; WIN32-NEXT: movzbl {{[0-9]+}}(%esp), %eax -; WIN32-NEXT: pushl %eax -; WIN32-NEXT: pushl $1072693248 # imm = 0x3FF00000 -; WIN32-NEXT: pushl $0 +; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) +; WIN32-NEXT: fld1 +; WIN32-NEXT: fstpl (%esp) ; WIN32-NEXT: calll _ldexp ; WIN32-NEXT: addl $12, %esp ; WIN32-NEXT: retl diff --git a/llvm/test/CodeGen/X86/llvm.frexp.ll b/llvm/test/CodeGen/X86/llvm.frexp.ll index bd0a1dce6946..d6038cebd458 100644 --- a/llvm/test/CodeGen/X86/llvm.frexp.ll +++ b/llvm/test/CodeGen/X86/llvm.frexp.ll @@ -2,24 +2,241 @@ ; RUN: llc -mtriple=x86_64-unknown-unknown < %s | FileCheck -check-prefixes=X64 %s ; RUN: llc -mtriple=i386-pc-win32 < %s | FileCheck -check-prefix=WIN32 %s -; FIXME -; define { half, i32 } @test_frexp_f16_i32(half %a) { -; %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) -; ret { half, i32 } %result -; } +define { half, i32 } @test_frexp_f16_i32(half %a) { +; X64-LABEL: test_frexp_f16_i32: +; X64: # %bb.0: +; X64-NEXT: subq $24, %rsp +; X64-NEXT: .cfi_def_cfa_offset 32 +; X64-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill +; X64-NEXT: callq __extendhfsf2@PLT +; X64-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: callq __truncsfhf2@PLT +; X64-NEXT: pextrw $0, %xmm0, %ecx +; X64-NEXT: movl %ecx, %eax +; X64-NEXT: andl $31744, %eax # imm = 0x7C00 +; X64-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload +; X64-NEXT: pextrw $0, %xmm0, %edx +; X64-NEXT: movl %edx, %esi +; X64-NEXT: andl $32767, %esi # imm = 0x7FFF +; X64-NEXT: cmpl $1024, %esi # imm = 0x400 +; X64-NEXT: cmovael %edx, %ecx +; X64-NEXT: cmovael %esi, %eax +; X64-NEXT: shrl $10, %eax +; X64-NEXT: leal -12(%rax), %edi +; X64-NEXT: cmpl $1024, %esi # imm = 0x400 +; X64-NEXT: cmovael %eax, %edi +; X64-NEXT: addl $-14, %edi +; X64-NEXT: andl $-31745, %ecx # imm = 0x83FF +; X64-NEXT: orl $14336, %ecx # imm = 0x3800 +; X64-NEXT: addl $-31744, %esi # imm = 0x8400 +; X64-NEXT: movzwl %si, %esi +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: cmpl $33792, %esi # imm = 0x8400 +; X64-NEXT: cmoval %edi, %eax +; X64-NEXT: cmovbel %edx, %ecx +; X64-NEXT: pinsrw $0, %ecx, %xmm0 +; X64-NEXT: addq $24, %rsp +; X64-NEXT: .cfi_def_cfa_offset 8 +; X64-NEXT: retq +; +; WIN32-LABEL: test_frexp_f16_i32: +; WIN32: # %bb.0: +; WIN32-NEXT: pushl %esi +; WIN32-NEXT: subl $16, %esp +; WIN32-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: movl %eax, (%esp) +; WIN32-NEXT: calll ___gnu_h2f_ieee +; WIN32-NEXT: fld %st(0) +; WIN32-NEXT: fmuls __real@4c000000 +; WIN32-NEXT: fstps {{[0-9]+}}(%esp) +; WIN32-NEXT: fsts {{[0-9]+}}(%esp) +; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edx +; WIN32-NEXT: movl %edx, %ecx +; WIN32-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF +; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 +; WIN32-NEXT: jb LBB0_1 +; WIN32-NEXT: # %bb.2: +; WIN32-NEXT: movl %ecx, %eax +; WIN32-NEXT: jmp LBB0_3 +; WIN32-NEXT: LBB0_1: +; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edx +; WIN32-NEXT: movl %edx, %eax +; WIN32-NEXT: andl $2139095040, %eax # imm = 0x7F800000 +; WIN32-NEXT: LBB0_3: +; WIN32-NEXT: shrl $23, %eax +; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 +; WIN32-NEXT: jae LBB0_5 +; WIN32-NEXT: # %bb.4: +; WIN32-NEXT: addl $-25, %eax +; WIN32-NEXT: LBB0_5: +; WIN32-NEXT: andl $-2139095041, %edx # imm = 0x807FFFFF +; WIN32-NEXT: orl $1056964608, %edx # imm = 0x3F000000 +; WIN32-NEXT: movl %edx, {{[0-9]+}}(%esp) +; WIN32-NEXT: addl $-126, %eax +; WIN32-NEXT: addl $-2139095040, %ecx # imm = 0x80800000 +; WIN32-NEXT: flds {{[0-9]+}}(%esp) +; WIN32-NEXT: xorl %esi, %esi +; WIN32-NEXT: cmpl $-2139095040, %ecx # imm = 0x80800000 +; WIN32-NEXT: jbe LBB0_7 +; WIN32-NEXT: # %bb.6: +; WIN32-NEXT: fstp %st(1) +; WIN32-NEXT: fldz +; WIN32-NEXT: LBB0_7: +; WIN32-NEXT: fstp %st(0) +; WIN32-NEXT: fstps (%esp) +; WIN32-NEXT: jbe LBB0_9 +; WIN32-NEXT: # %bb.8: +; WIN32-NEXT: movl %eax, %esi +; WIN32-NEXT: LBB0_9: +; WIN32-NEXT: calll ___gnu_f2h_ieee +; WIN32-NEXT: movl %esi, %edx +; WIN32-NEXT: addl $16, %esp +; WIN32-NEXT: popl %esi +; WIN32-NEXT: retl + %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + ret { half, i32 } %result +} -; define half @test_frexp_f16_i32_only_use_fract(half %a) { -; %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) -; %result.0 = extractvalue { half, i32 } %result, 0 -; ret half %result.0 -; } +define half @test_frexp_f16_i32_only_use_fract(half %a) { +; X64-LABEL: test_frexp_f16_i32_only_use_fract: +; X64: # %bb.0: +; X64-NEXT: subq $24, %rsp +; X64-NEXT: .cfi_def_cfa_offset 32 +; X64-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill +; X64-NEXT: callq __extendhfsf2@PLT +; X64-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: callq __truncsfhf2@PLT +; X64-NEXT: pextrw $0, %xmm0, %eax +; X64-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload +; X64-NEXT: pextrw $0, %xmm0, %ecx +; X64-NEXT: movl %ecx, %edx +; X64-NEXT: andl $32767, %edx # imm = 0x7FFF +; X64-NEXT: cmpl $1024, %edx # imm = 0x400 +; X64-NEXT: cmovael %ecx, %eax +; X64-NEXT: andl $-31745, %eax # imm = 0x83FF +; X64-NEXT: orl $14336, %eax # imm = 0x3800 +; X64-NEXT: addl $-31744, %edx # imm = 0x8400 +; X64-NEXT: movzwl %dx, %edx +; X64-NEXT: cmpl $33792, %edx # imm = 0x8400 +; X64-NEXT: cmovbel %ecx, %eax +; X64-NEXT: pinsrw $0, %eax, %xmm0 +; X64-NEXT: addq $24, %rsp +; X64-NEXT: .cfi_def_cfa_offset 8 +; X64-NEXT: retq +; +; WIN32-LABEL: test_frexp_f16_i32_only_use_fract: +; WIN32: # %bb.0: +; WIN32-NEXT: subl $16, %esp +; WIN32-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: movl %eax, (%esp) +; WIN32-NEXT: calll ___gnu_h2f_ieee +; WIN32-NEXT: fsts {{[0-9]+}}(%esp) +; WIN32-NEXT: fld %st(0) +; WIN32-NEXT: fmuls __real@4c000000 +; WIN32-NEXT: fstps {{[0-9]+}}(%esp) +; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx +; WIN32-NEXT: movl %ecx, %eax +; WIN32-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF +; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 +; WIN32-NEXT: jae LBB1_2 +; WIN32-NEXT: # %bb.1: +; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx +; WIN32-NEXT: LBB1_2: +; WIN32-NEXT: andl $-2139095041, %ecx # imm = 0x807FFFFF +; WIN32-NEXT: orl $1056964608, %ecx # imm = 0x3F000000 +; WIN32-NEXT: movl %ecx, {{[0-9]+}}(%esp) +; WIN32-NEXT: addl $-2139095040, %eax # imm = 0x80800000 +; WIN32-NEXT: flds {{[0-9]+}}(%esp) +; WIN32-NEXT: cmpl $-2139095040, %eax # imm = 0x80800000 +; WIN32-NEXT: jbe LBB1_4 +; WIN32-NEXT: # %bb.3: +; WIN32-NEXT: fstp %st(1) +; WIN32-NEXT: fldz +; WIN32-NEXT: LBB1_4: +; WIN32-NEXT: fstp %st(0) +; WIN32-NEXT: fstps (%esp) +; WIN32-NEXT: calll ___gnu_f2h_ieee +; WIN32-NEXT: addl $16, %esp +; WIN32-NEXT: retl + %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + %result.0 = extractvalue { half, i32 } %result, 0 + ret half %result.0 +} -; define i32 @test_frexp_f16_i32_only_use_exp(half %a) { -; %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) -; %result.0 = extractvalue { half, i32 } %result, 1 -; ret i32 %result.0 -; } +define i32 @test_frexp_f16_i32_only_use_exp(half %a) { +; X64-LABEL: test_frexp_f16_i32_only_use_exp: +; X64: # %bb.0: +; X64-NEXT: subq $24, %rsp +; X64-NEXT: .cfi_def_cfa_offset 32 +; X64-NEXT: movaps %xmm0, (%rsp) # 16-byte Spill +; X64-NEXT: callq __extendhfsf2@PLT +; X64-NEXT: mulss {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; X64-NEXT: callq __truncsfhf2@PLT +; X64-NEXT: pextrw $0, %xmm0, %eax +; X64-NEXT: andl $31744, %eax # imm = 0x7C00 +; X64-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload +; X64-NEXT: pextrw $0, %xmm0, %ecx +; X64-NEXT: andl $32767, %ecx # imm = 0x7FFF +; X64-NEXT: cmpl $1024, %ecx # imm = 0x400 +; X64-NEXT: cmovael %ecx, %eax +; X64-NEXT: shrl $10, %eax +; X64-NEXT: leal -12(%rax), %edx +; X64-NEXT: cmpl $1024, %ecx # imm = 0x400 +; X64-NEXT: cmovael %eax, %edx +; X64-NEXT: addl $-14, %edx +; X64-NEXT: addl $-31744, %ecx # imm = 0x8400 +; X64-NEXT: movzwl %cx, %ecx +; X64-NEXT: xorl %eax, %eax +; X64-NEXT: cmpl $33792, %ecx # imm = 0x8400 +; X64-NEXT: cmoval %edx, %eax +; X64-NEXT: addq $24, %rsp +; X64-NEXT: .cfi_def_cfa_offset 8 +; X64-NEXT: retq +; +; WIN32-LABEL: test_frexp_f16_i32_only_use_exp: +; WIN32: # %bb.0: +; WIN32-NEXT: subl $8, %esp +; WIN32-NEXT: movzwl {{[0-9]+}}(%esp), %eax +; WIN32-NEXT: pushl %eax +; WIN32-NEXT: calll ___gnu_h2f_ieee +; WIN32-NEXT: addl $4, %esp +; WIN32-NEXT: fld %st(0) +; WIN32-NEXT: fmuls __real@4c000000 +; WIN32-NEXT: fstps {{[0-9]+}}(%esp) +; WIN32-NEXT: fstps (%esp) +; WIN32-NEXT: movl $2147483647, %ecx # imm = 0x7FFFFFFF +; WIN32-NEXT: andl (%esp), %ecx +; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 +; WIN32-NEXT: jb LBB2_1 +; WIN32-NEXT: # %bb.2: +; WIN32-NEXT: movl %ecx, %edx +; WIN32-NEXT: jmp LBB2_3 +; WIN32-NEXT: LBB2_1: +; WIN32-NEXT: movl $2139095040, %edx # imm = 0x7F800000 +; WIN32-NEXT: andl {{[0-9]+}}(%esp), %edx +; WIN32-NEXT: LBB2_3: +; WIN32-NEXT: shrl $23, %edx +; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 +; WIN32-NEXT: jae LBB2_5 +; WIN32-NEXT: # %bb.4: +; WIN32-NEXT: addl $-25, %edx +; WIN32-NEXT: LBB2_5: +; WIN32-NEXT: addl $-2139095040, %ecx # imm = 0x80800000 +; WIN32-NEXT: xorl %eax, %eax +; WIN32-NEXT: cmpl $-2139095040, %ecx # imm = 0x80800000 +; WIN32-NEXT: jbe LBB2_7 +; WIN32-NEXT: # %bb.6: +; WIN32-NEXT: addl $-126, %edx +; WIN32-NEXT: movl %edx, %eax +; WIN32-NEXT: LBB2_7: +; WIN32-NEXT: addl $8, %esp +; WIN32-NEXT: retl + %result = call { half, i32 } @llvm.frexp.f16.i32(half %a) + %result.0 = extractvalue { half, i32 } %result, 1 + ret i32 %result.0 +} +; FIXME ; define { <2 x half>, <2 x i32> } @test_frexp_v2f16_v2i32(<2 x half> %a) { ; %result = call { <2 x half>, <2 x i32> } @llvm.frexp.v2f16.v2i32(<2 x half> %a) ; ret { <2 x half>, <2 x i32> } %result @@ -61,21 +278,21 @@ define { float, i32 } @test_frexp_f32_i32(float %a) { ; WIN32-NEXT: movl %eax, %ecx ; WIN32-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 -; WIN32-NEXT: jb LBB0_1 +; WIN32-NEXT: jb LBB3_1 ; WIN32-NEXT: # %bb.2: ; WIN32-NEXT: movl %ecx, %edx -; WIN32-NEXT: jmp LBB0_3 -; WIN32-NEXT: LBB0_1: +; WIN32-NEXT: jmp LBB3_3 +; WIN32-NEXT: LBB3_1: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: movl %eax, %edx ; WIN32-NEXT: andl $2139095040, %edx # imm = 0x7F800000 -; WIN32-NEXT: LBB0_3: +; WIN32-NEXT: LBB3_3: ; WIN32-NEXT: shrl $23, %edx ; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 -; WIN32-NEXT: jae LBB0_5 +; WIN32-NEXT: jae LBB3_5 ; WIN32-NEXT: # %bb.4: ; WIN32-NEXT: addl $-25, %edx -; WIN32-NEXT: LBB0_5: +; WIN32-NEXT: LBB3_5: ; WIN32-NEXT: andl $-2139095041, %eax # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %eax # imm = 0x3F000000 ; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) @@ -83,16 +300,16 @@ define { float, i32 } @test_frexp_f32_i32(float %a) { ; WIN32-NEXT: addl $-2139095040, %ecx # imm = 0x80800000 ; WIN32-NEXT: xorl %eax, %eax ; WIN32-NEXT: cmpl $-2139095040, %ecx # imm = 0x80800000 -; WIN32-NEXT: jbe LBB0_7 +; WIN32-NEXT: jbe LBB3_7 ; WIN32-NEXT: # %bb.6: ; WIN32-NEXT: movl %edx, %eax -; WIN32-NEXT: LBB0_7: +; WIN32-NEXT: LBB3_7: ; WIN32-NEXT: flds {{[0-9]+}}(%esp) -; WIN32-NEXT: jbe LBB0_9 +; WIN32-NEXT: jbe LBB3_9 ; WIN32-NEXT: # %bb.8: ; WIN32-NEXT: fstp %st(1) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB0_9: +; WIN32-NEXT: LBB3_9: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: addl $12, %esp ; WIN32-NEXT: retl @@ -123,21 +340,21 @@ define float @test_frexp_f32_i32_only_use_fract(float %a) { ; WIN32-NEXT: movl %ecx, %eax ; WIN32-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 -; WIN32-NEXT: jae LBB1_2 +; WIN32-NEXT: jae LBB4_2 ; WIN32-NEXT: # %bb.1: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; WIN32-NEXT: LBB1_2: +; WIN32-NEXT: LBB4_2: ; WIN32-NEXT: andl $-2139095041, %ecx # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %ecx # imm = 0x3F000000 ; WIN32-NEXT: movl %ecx, {{[0-9]+}}(%esp) ; WIN32-NEXT: addl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: cmpl $-2139095040, %eax # imm = 0x80800000 -; WIN32-NEXT: jbe LBB1_4 +; WIN32-NEXT: jbe LBB4_4 ; WIN32-NEXT: # %bb.3: ; WIN32-NEXT: fstp %st(1) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB1_4: +; WIN32-NEXT: LBB4_4: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: addl $12, %esp ; WIN32-NEXT: retl @@ -169,28 +386,28 @@ define i32 @test_frexp_f32_i32_only_use_exp(float %a) { ; WIN32-NEXT: movl $2147483647, %ecx # imm = 0x7FFFFFFF ; WIN32-NEXT: andl (%esp), %ecx ; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 -; WIN32-NEXT: jb LBB2_1 +; WIN32-NEXT: jb LBB5_1 ; WIN32-NEXT: # %bb.2: ; WIN32-NEXT: movl %ecx, %edx -; WIN32-NEXT: jmp LBB2_3 -; WIN32-NEXT: LBB2_1: +; WIN32-NEXT: jmp LBB5_3 +; WIN32-NEXT: LBB5_1: ; WIN32-NEXT: movl $2139095040, %edx # imm = 0x7F800000 ; WIN32-NEXT: andl {{[0-9]+}}(%esp), %edx -; WIN32-NEXT: LBB2_3: +; WIN32-NEXT: LBB5_3: ; WIN32-NEXT: shrl $23, %edx ; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 -; WIN32-NEXT: jae LBB2_5 +; WIN32-NEXT: jae LBB5_5 ; WIN32-NEXT: # %bb.4: ; WIN32-NEXT: addl $-25, %edx -; WIN32-NEXT: LBB2_5: +; WIN32-NEXT: LBB5_5: ; WIN32-NEXT: addl $-2139095040, %ecx # imm = 0x80800000 ; WIN32-NEXT: xorl %eax, %eax ; WIN32-NEXT: cmpl $-2139095040, %ecx # imm = 0x80800000 -; WIN32-NEXT: jbe LBB2_7 +; WIN32-NEXT: jbe LBB5_7 ; WIN32-NEXT: # %bb.6: ; WIN32-NEXT: addl $-126, %edx ; WIN32-NEXT: movl %edx, %eax -; WIN32-NEXT: LBB2_7: +; WIN32-NEXT: LBB5_7: ; WIN32-NEXT: addl $8, %esp ; WIN32-NEXT: retl %result = call { float, i32 } @llvm.frexp.f32.i32(float %a) @@ -275,15 +492,15 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) { ; WIN32-NEXT: movl %ebx, %eax ; WIN32-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 -; WIN32-NEXT: jb LBB3_1 +; WIN32-NEXT: jb LBB6_1 ; WIN32-NEXT: # %bb.2: ; WIN32-NEXT: movl %eax, %ecx -; WIN32-NEXT: jmp LBB3_3 -; WIN32-NEXT: LBB3_1: +; WIN32-NEXT: jmp LBB6_3 +; WIN32-NEXT: LBB6_1: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ebx ; WIN32-NEXT: movl %ebx, %ecx ; WIN32-NEXT: andl $2139095040, %ecx # imm = 0x7F800000 -; WIN32-NEXT: LBB3_3: +; WIN32-NEXT: LBB6_3: ; WIN32-NEXT: shrl $23, %ecx ; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) @@ -291,24 +508,24 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) { ; WIN32-NEXT: fmul %st(3), %st ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) ; WIN32-NEXT: fsts {{[0-9]+}}(%esp) -; WIN32-NEXT: jae LBB3_5 +; WIN32-NEXT: jae LBB6_5 ; WIN32-NEXT: # %bb.4: ; WIN32-NEXT: addl $-25, %ecx -; WIN32-NEXT: LBB3_5: +; WIN32-NEXT: LBB6_5: ; WIN32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx ; WIN32-NEXT: movl %ecx, %edx ; WIN32-NEXT: andl $2147483647, %edx # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 -; WIN32-NEXT: jb LBB3_6 +; WIN32-NEXT: jb LBB6_6 ; WIN32-NEXT: # %bb.7: ; WIN32-NEXT: movl %edx, %esi -; WIN32-NEXT: jmp LBB3_8 -; WIN32-NEXT: LBB3_6: +; WIN32-NEXT: jmp LBB6_8 +; WIN32-NEXT: LBB6_6: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx ; WIN32-NEXT: movl %ecx, %esi ; WIN32-NEXT: andl $2139095040, %esi # imm = 0x7F800000 -; WIN32-NEXT: LBB3_8: +; WIN32-NEXT: LBB6_8: ; WIN32-NEXT: shrl $23, %esi ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) @@ -316,24 +533,24 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) { ; WIN32-NEXT: fmul %st(4), %st ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) ; WIN32-NEXT: fsts {{[0-9]+}}(%esp) -; WIN32-NEXT: jae LBB3_10 +; WIN32-NEXT: jae LBB6_10 ; WIN32-NEXT: # %bb.9: ; WIN32-NEXT: addl $-25, %esi -; WIN32-NEXT: LBB3_10: +; WIN32-NEXT: LBB6_10: ; WIN32-NEXT: movl %esi, (%esp) # 4-byte Spill ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edi ; WIN32-NEXT: movl %edi, %ebp ; WIN32-NEXT: andl $2147483647, %ebp # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %ebp # imm = 0x800000 -; WIN32-NEXT: jb LBB3_11 +; WIN32-NEXT: jb LBB6_11 ; WIN32-NEXT: # %bb.12: ; WIN32-NEXT: movl %ebp, %esi -; WIN32-NEXT: jmp LBB3_13 -; WIN32-NEXT: LBB3_11: +; WIN32-NEXT: jmp LBB6_13 +; WIN32-NEXT: LBB6_11: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edi ; WIN32-NEXT: movl %edi, %esi ; WIN32-NEXT: andl $2139095040, %esi # imm = 0x7F800000 -; WIN32-NEXT: LBB3_13: +; WIN32-NEXT: LBB6_13: ; WIN32-NEXT: shrl $23, %esi ; WIN32-NEXT: cmpl $8388608, %ebp # imm = 0x800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) @@ -344,30 +561,30 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) { ; WIN32-NEXT: fsts {{[0-9]+}}(%esp) ; WIN32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; WIN32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill -; WIN32-NEXT: jae LBB3_15 +; WIN32-NEXT: jae LBB6_15 ; WIN32-NEXT: # %bb.14: ; WIN32-NEXT: addl $-25, %esi -; WIN32-NEXT: LBB3_15: +; WIN32-NEXT: LBB6_15: ; WIN32-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: movl %eax, %edx ; WIN32-NEXT: andl $2147483647, %edx # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 -; WIN32-NEXT: jb LBB3_16 +; WIN32-NEXT: jb LBB6_16 ; WIN32-NEXT: # %bb.17: ; WIN32-NEXT: movl %edx, %esi -; WIN32-NEXT: jmp LBB3_18 -; WIN32-NEXT: LBB3_16: +; WIN32-NEXT: jmp LBB6_18 +; WIN32-NEXT: LBB6_16: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: movl %eax, %esi ; WIN32-NEXT: andl $2139095040, %esi # imm = 0x7F800000 -; WIN32-NEXT: LBB3_18: +; WIN32-NEXT: LBB6_18: ; WIN32-NEXT: shrl $23, %esi ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 -; WIN32-NEXT: jae LBB3_20 +; WIN32-NEXT: jae LBB6_20 ; WIN32-NEXT: # %bb.19: ; WIN32-NEXT: addl $-25, %esi -; WIN32-NEXT: LBB3_20: +; WIN32-NEXT: LBB6_20: ; WIN32-NEXT: andl $-2139095041, %eax # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %eax # imm = 0x3F000000 ; WIN32-NEXT: movl %eax, {{[0-9]+}}(%esp) @@ -389,61 +606,61 @@ define { <4 x float>, <4 x i32> } @test_frexp_v4f32_v4i32(<4 x float> %a) { ; WIN32-NEXT: xorl %edi, %edi ; WIN32-NEXT: cmpl $-2139095040, %edx # imm = 0x80800000 ; WIN32-NEXT: movl $0, %ebx -; WIN32-NEXT: jbe LBB3_22 +; WIN32-NEXT: jbe LBB6_22 ; WIN32-NEXT: # %bb.21: ; WIN32-NEXT: movl %esi, %ebx -; WIN32-NEXT: LBB3_22: +; WIN32-NEXT: LBB6_22: ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload -; WIN32-NEXT: jbe LBB3_24 +; WIN32-NEXT: jbe LBB6_24 ; WIN32-NEXT: # %bb.23: ; WIN32-NEXT: fstp %st(1) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB3_24: +; WIN32-NEXT: LBB6_24: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: addl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: movl $0, %edx ; WIN32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload -; WIN32-NEXT: jbe LBB3_26 +; WIN32-NEXT: jbe LBB6_26 ; WIN32-NEXT: # %bb.25: ; WIN32-NEXT: movl %ecx, %edx -; WIN32-NEXT: LBB3_26: +; WIN32-NEXT: LBB6_26: ; WIN32-NEXT: flds {{[0-9]+}}(%esp) -; WIN32-NEXT: jbe LBB3_28 +; WIN32-NEXT: jbe LBB6_28 ; WIN32-NEXT: # %bb.27: ; WIN32-NEXT: fstp %st(3) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB3_28: +; WIN32-NEXT: LBB6_28: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload ; WIN32-NEXT: addl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: movl $0, %ecx -; WIN32-NEXT: jbe LBB3_30 +; WIN32-NEXT: jbe LBB6_30 ; WIN32-NEXT: # %bb.29: ; WIN32-NEXT: movl (%esp), %ecx # 4-byte Reload -; WIN32-NEXT: LBB3_30: +; WIN32-NEXT: LBB6_30: ; WIN32-NEXT: flds {{[0-9]+}}(%esp) -; WIN32-NEXT: jbe LBB3_32 +; WIN32-NEXT: jbe LBB6_32 ; WIN32-NEXT: # %bb.31: ; WIN32-NEXT: fstp %st(2) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB3_32: +; WIN32-NEXT: LBB6_32: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: addl $-2139095040, %ebp # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %ebp # imm = 0x80800000 -; WIN32-NEXT: jbe LBB3_34 +; WIN32-NEXT: jbe LBB6_34 ; WIN32-NEXT: # %bb.33: ; WIN32-NEXT: movl %esi, %edi -; WIN32-NEXT: LBB3_34: +; WIN32-NEXT: LBB6_34: ; WIN32-NEXT: flds {{[0-9]+}}(%esp) -; WIN32-NEXT: jbe LBB3_36 +; WIN32-NEXT: jbe LBB6_36 ; WIN32-NEXT: # %bb.35: ; WIN32-NEXT: fstp %st(4) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB3_36: +; WIN32-NEXT: LBB6_36: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: movl %edi, 28(%eax) ; WIN32-NEXT: movl %ecx, 24(%eax) @@ -531,10 +748,10 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) { ; WIN32-NEXT: movl %ecx, %eax ; WIN32-NEXT: andl $2147483647, %eax # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 -; WIN32-NEXT: jae LBB4_2 +; WIN32-NEXT: jae LBB7_2 ; WIN32-NEXT: # %bb.1: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ecx -; WIN32-NEXT: LBB4_2: +; WIN32-NEXT: LBB7_2: ; WIN32-NEXT: andl $-2139095041, %ecx # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %ecx # imm = 0x3F000000 ; WIN32-NEXT: movl %ecx, {{[0-9]+}}(%esp) @@ -542,10 +759,10 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) { ; WIN32-NEXT: movl %edx, %ecx ; WIN32-NEXT: andl $2147483647, %ecx # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %ecx # imm = 0x800000 -; WIN32-NEXT: jae LBB4_4 +; WIN32-NEXT: jae LBB7_4 ; WIN32-NEXT: # %bb.3: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edx -; WIN32-NEXT: LBB4_4: +; WIN32-NEXT: LBB7_4: ; WIN32-NEXT: andl $-2139095041, %edx # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %edx # imm = 0x3F000000 ; WIN32-NEXT: movl %edx, {{[0-9]+}}(%esp) @@ -553,10 +770,10 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) { ; WIN32-NEXT: movl %esi, %edx ; WIN32-NEXT: andl $2147483647, %edx # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 -; WIN32-NEXT: jae LBB4_6 +; WIN32-NEXT: jae LBB7_6 ; WIN32-NEXT: # %bb.5: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %esi -; WIN32-NEXT: LBB4_6: +; WIN32-NEXT: LBB7_6: ; WIN32-NEXT: andl $-2139095041, %esi # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %esi # imm = 0x3F000000 ; WIN32-NEXT: movl %esi, {{[0-9]+}}(%esp) @@ -564,49 +781,49 @@ define <4 x float> @test_frexp_v4f32_v4i32_only_use_fract(<4 x float> %a) { ; WIN32-NEXT: movl %edi, %esi ; WIN32-NEXT: andl $2147483647, %esi # imm = 0x7FFFFFFF ; WIN32-NEXT: cmpl $8388608, %esi # imm = 0x800000 -; WIN32-NEXT: jae LBB4_8 +; WIN32-NEXT: jae LBB7_8 ; WIN32-NEXT: # %bb.7: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edi -; WIN32-NEXT: LBB4_8: +; WIN32-NEXT: LBB7_8: ; WIN32-NEXT: andl $-2139095041, %edi # imm = 0x807FFFFF ; WIN32-NEXT: orl $1056964608, %edi # imm = 0x3F000000 ; WIN32-NEXT: movl %edi, {{[0-9]+}}(%esp) ; WIN32-NEXT: addl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: cmpl $-2139095040, %eax # imm = 0x80800000 -; WIN32-NEXT: jbe LBB4_10 +; WIN32-NEXT: jbe LBB7_10 ; WIN32-NEXT: # %bb.9: ; WIN32-NEXT: fstp %st(1) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB4_10: +; WIN32-NEXT: LBB7_10: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: addl $-2139095040, %ecx # imm = 0x80800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: cmpl $-2139095040, %ecx # imm = 0x80800000 -; WIN32-NEXT: jbe LBB4_12 +; WIN32-NEXT: jbe LBB7_12 ; WIN32-NEXT: # %bb.11: ; WIN32-NEXT: fstp %st(3) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB4_12: +; WIN32-NEXT: LBB7_12: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: addl $-2139095040, %edx # imm = 0x80800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: cmpl $-2139095040, %edx # imm = 0x80800000 -; WIN32-NEXT: jbe LBB4_14 +; WIN32-NEXT: jbe LBB7_14 ; WIN32-NEXT: # %bb.13: ; WIN32-NEXT: fstp %st(4) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB4_14: +; WIN32-NEXT: LBB7_14: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: addl $-2139095040, %esi # imm = 0x80800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) ; WIN32-NEXT: cmpl $-2139095040, %esi # imm = 0x80800000 -; WIN32-NEXT: jbe LBB4_16 +; WIN32-NEXT: jbe LBB7_16 ; WIN32-NEXT: # %bb.15: ; WIN32-NEXT: fstp %st(2) ; WIN32-NEXT: fldz -; WIN32-NEXT: LBB4_16: +; WIN32-NEXT: LBB7_16: ; WIN32-NEXT: fstp %st(0) ; WIN32-NEXT: fxch %st(1) ; WIN32-NEXT: fstps 12(%eax) @@ -673,14 +890,14 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) { ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: andl %edx, %eax ; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 -; WIN32-NEXT: jb LBB5_1 +; WIN32-NEXT: jb LBB8_1 ; WIN32-NEXT: # %bb.2: ; WIN32-NEXT: movl %eax, %esi -; WIN32-NEXT: jmp LBB5_3 -; WIN32-NEXT: LBB5_1: +; WIN32-NEXT: jmp LBB8_3 +; WIN32-NEXT: LBB8_1: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %esi ; WIN32-NEXT: andl %ecx, %esi -; WIN32-NEXT: LBB5_3: +; WIN32-NEXT: LBB8_3: ; WIN32-NEXT: shrl $23, %esi ; WIN32-NEXT: cmpl $8388608, %eax # imm = 0x800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) @@ -688,21 +905,21 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) { ; WIN32-NEXT: fmul %st(2), %st ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) -; WIN32-NEXT: jae LBB5_5 +; WIN32-NEXT: jae LBB8_5 ; WIN32-NEXT: # %bb.4: ; WIN32-NEXT: addl $-25, %esi -; WIN32-NEXT: LBB5_5: +; WIN32-NEXT: LBB8_5: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %edi ; WIN32-NEXT: andl %edx, %edi ; WIN32-NEXT: cmpl $8388608, %edi # imm = 0x800000 -; WIN32-NEXT: jb LBB5_6 +; WIN32-NEXT: jb LBB8_6 ; WIN32-NEXT: # %bb.7: ; WIN32-NEXT: movl %edi, %ebx -; WIN32-NEXT: jmp LBB5_8 -; WIN32-NEXT: LBB5_6: +; WIN32-NEXT: jmp LBB8_8 +; WIN32-NEXT: LBB8_6: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ebx ; WIN32-NEXT: andl %ecx, %ebx -; WIN32-NEXT: LBB5_8: +; WIN32-NEXT: LBB8_8: ; WIN32-NEXT: shrl $23, %ebx ; WIN32-NEXT: cmpl $8388608, %edi # imm = 0x800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) @@ -710,22 +927,22 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) { ; WIN32-NEXT: fmul %st(2), %st ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) -; WIN32-NEXT: jae LBB5_10 +; WIN32-NEXT: jae LBB8_10 ; WIN32-NEXT: # %bb.9: ; WIN32-NEXT: addl $-25, %ebx -; WIN32-NEXT: LBB5_10: +; WIN32-NEXT: LBB8_10: ; WIN32-NEXT: movl %esi, (%esp) # 4-byte Spill ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %ebp ; WIN32-NEXT: andl %edx, %ebp ; WIN32-NEXT: cmpl $8388608, %ebp # imm = 0x800000 -; WIN32-NEXT: jb LBB5_11 +; WIN32-NEXT: jb LBB8_11 ; WIN32-NEXT: # %bb.12: ; WIN32-NEXT: movl %ebp, %esi -; WIN32-NEXT: jmp LBB5_13 -; WIN32-NEXT: LBB5_11: +; WIN32-NEXT: jmp LBB8_13 +; WIN32-NEXT: LBB8_11: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %esi ; WIN32-NEXT: andl %ecx, %esi -; WIN32-NEXT: LBB5_13: +; WIN32-NEXT: LBB8_13: ; WIN32-NEXT: shrl $23, %esi ; WIN32-NEXT: cmpl $8388608, %ebp # imm = 0x800000 ; WIN32-NEXT: flds {{[0-9]+}}(%esp) @@ -733,59 +950,59 @@ define <4 x i32> @test_frexp_v4f32_v4i32_only_use_exp(<4 x float> %a) { ; WIN32-NEXT: fxch %st(1) ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) ; WIN32-NEXT: fstps {{[0-9]+}}(%esp) -; WIN32-NEXT: jae LBB5_15 +; WIN32-NEXT: jae LBB8_15 ; WIN32-NEXT: # %bb.14: ; WIN32-NEXT: addl $-25, %esi -; WIN32-NEXT: LBB5_15: +; WIN32-NEXT: LBB8_15: ; WIN32-NEXT: andl {{[0-9]+}}(%esp), %edx ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 -; WIN32-NEXT: jb LBB5_16 +; WIN32-NEXT: jb LBB8_16 ; WIN32-NEXT: # %bb.17: ; WIN32-NEXT: movl %edx, %ecx -; WIN32-NEXT: jmp LBB5_18 -; WIN32-NEXT: LBB5_16: +; WIN32-NEXT: jmp LBB8_18 +; WIN32-NEXT: LBB8_16: ; WIN32-NEXT: andl {{[0-9]+}}(%esp), %ecx -; WIN32-NEXT: LBB5_18: +; WIN32-NEXT: LBB8_18: ; WIN32-NEXT: shrl $23, %ecx ; WIN32-NEXT: cmpl $8388608, %edx # imm = 0x800000 -; WIN32-NEXT: jae LBB5_20 +; WIN32-NEXT: jae LBB8_20 ; WIN32-NEXT: # %bb.19: ; WIN32-NEXT: addl $-25, %ecx -; WIN32-NEXT: LBB5_20: +; WIN32-NEXT: LBB8_20: ; WIN32-NEXT: addl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %eax # imm = 0x80800000 ; WIN32-NEXT: movl $0, %eax -; WIN32-NEXT: jbe LBB5_22 +; WIN32-NEXT: jbe LBB8_22 ; WIN32-NEXT: # %bb.21: ; WIN32-NEXT: movl (%esp), %eax # 4-byte Reload ; WIN32-NEXT: addl $-126, %eax -; WIN32-NEXT: LBB5_22: +; WIN32-NEXT: LBB8_22: ; WIN32-NEXT: movl %eax, (%esp) # 4-byte Spill ; WIN32-NEXT: addl $-2139095040, %edi # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %edi # imm = 0x80800000 ; WIN32-NEXT: movl $0, %edi -; WIN32-NEXT: jbe LBB5_24 +; WIN32-NEXT: jbe LBB8_24 ; WIN32-NEXT: # %bb.23: ; WIN32-NEXT: addl $-126, %ebx ; WIN32-NEXT: movl %ebx, %edi -; WIN32-NEXT: LBB5_24: +; WIN32-NEXT: LBB8_24: ; WIN32-NEXT: movl {{[0-9]+}}(%esp), %eax ; WIN32-NEXT: addl $-2139095040, %ebp # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %ebp # imm = 0x80800000 ; WIN32-NEXT: movl $0, %ebx -; WIN32-NEXT: jbe LBB5_26 +; WIN32-NEXT: jbe LBB8_26 ; WIN32-NEXT: # %bb.25: ; WIN32-NEXT: addl $-126, %esi ; WIN32-NEXT: movl %esi, %ebx -; WIN32-NEXT: LBB5_26: +; WIN32-NEXT: LBB8_26: ; WIN32-NEXT: addl $-2139095040, %edx # imm = 0x80800000 ; WIN32-NEXT: cmpl $-2139095040, %edx # imm = 0x80800000 ; WIN32-NEXT: movl $0, %edx -; WIN32-NEXT: jbe LBB5_28 +; WIN32-NEXT: jbe LBB8_28 ; WIN32-NEXT: # %bb.27: ; WIN32-NEXT: addl $-126, %ecx ; WIN32-NEXT: movl %ecx, %edx -; WIN32-NEXT: LBB5_28: +; WIN32-NEXT: LBB8_28: ; WIN32-NEXT: movl %edx, 12(%eax) ; WIN32-NEXT: movl %ebx, 8(%eax) ; WIN32-NEXT: movl %edi, 4(%eax) diff --git a/llvm/test/CodeGen/X86/memset64-on-x86-32.ll b/llvm/test/CodeGen/X86/memset64-on-x86-32.ll index c6eecdcdf99c..480a0970bd39 100644 --- a/llvm/test/CodeGen/X86/memset64-on-x86-32.ll +++ b/llvm/test/CodeGen/X86/memset64-on-x86-32.ll @@ -18,26 +18,17 @@ define void @bork(ptr nocapture align 4 %dst) nounwind { ; SLOW_32-LABEL: bork: ; SLOW_32: # %bb.0: ; SLOW_32-NEXT: movl {{[0-9]+}}(%esp), %eax -; SLOW_32-NEXT: movl $0, 4(%eax) -; SLOW_32-NEXT: movl $0, (%eax) -; SLOW_32-NEXT: movl $0, 12(%eax) -; SLOW_32-NEXT: movl $0, 8(%eax) -; SLOW_32-NEXT: movl $0, 20(%eax) -; SLOW_32-NEXT: movl $0, 16(%eax) -; SLOW_32-NEXT: movl $0, 28(%eax) -; SLOW_32-NEXT: movl $0, 24(%eax) -; SLOW_32-NEXT: movl $0, 36(%eax) -; SLOW_32-NEXT: movl $0, 32(%eax) -; SLOW_32-NEXT: movl $0, 44(%eax) -; SLOW_32-NEXT: movl $0, 40(%eax) -; SLOW_32-NEXT: movl $0, 52(%eax) -; SLOW_32-NEXT: movl $0, 48(%eax) -; SLOW_32-NEXT: movl $0, 60(%eax) -; SLOW_32-NEXT: movl $0, 56(%eax) -; SLOW_32-NEXT: movl $0, 68(%eax) -; SLOW_32-NEXT: movl $0, 64(%eax) -; SLOW_32-NEXT: movl $0, 76(%eax) -; SLOW_32-NEXT: movl $0, 72(%eax) +; SLOW_32-NEXT: xorps %xmm0, %xmm0 +; SLOW_32-NEXT: movsd %xmm0, 72(%eax) +; SLOW_32-NEXT: movsd %xmm0, 64(%eax) +; SLOW_32-NEXT: movsd %xmm0, 56(%eax) +; SLOW_32-NEXT: movsd %xmm0, 48(%eax) +; SLOW_32-NEXT: movsd %xmm0, 40(%eax) +; SLOW_32-NEXT: movsd %xmm0, 32(%eax) +; SLOW_32-NEXT: movsd %xmm0, 24(%eax) +; SLOW_32-NEXT: movsd %xmm0, 16(%eax) +; SLOW_32-NEXT: movsd %xmm0, 8(%eax) +; SLOW_32-NEXT: movsd %xmm0, (%eax) ; SLOW_32-NEXT: retl ; ; SLOW_64-LABEL: bork: diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll index 7dd4af768979..5cd0c232de44 100644 --- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll +++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll @@ -2010,7 +2010,7 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind "min-legal-v ; CHECK: # %bb.0: ; CHECK-NEXT: vpsllw $4, %ymm0, %ymm1 ; CHECK-NEXT: vpsrlw $4, %ymm0, %ymm0 -; CHECK-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; CHECK-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; CHECK-NEXT: retq %shl = shl <32 x i8> %a, %lshr = lshr <32 x i8> %a, @@ -2023,7 +2023,7 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind "min-le ; CHECK: # %bb.0: ; CHECK-NEXT: vpsllw $4, %ymm0, %ymm1 ; CHECK-NEXT: vpsrlw $4, %ymm0, %ymm0 -; CHECK-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; CHECK-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; CHECK-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0 ; CHECK-NEXT: retq %shl = shl <32 x i8> %a, diff --git a/llvm/test/CodeGen/X86/nocx16.ll b/llvm/test/CodeGen/X86/nocx16.ll deleted file mode 100644 index ec8e6b2c8c6a..000000000000 --- a/llvm/test/CodeGen/X86/nocx16.ll +++ /dev/null @@ -1,21 +0,0 @@ -; RUN: llc < %s -mtriple=x86_64-- -mcpu=corei7 -mattr=-cx16 | FileCheck %s -define void @test(ptr %a) nounwind { -entry: -; CHECK: __sync_val_compare_and_swap_16 - %0 = cmpxchg ptr %a, i128 1, i128 1 seq_cst seq_cst -; CHECK: __sync_lock_test_and_set_16 - %1 = atomicrmw xchg ptr %a, i128 1 seq_cst -; CHECK: __sync_fetch_and_add_16 - %2 = atomicrmw add ptr %a, i128 1 seq_cst -; CHECK: __sync_fetch_and_sub_16 - %3 = atomicrmw sub ptr %a, i128 1 seq_cst -; CHECK: __sync_fetch_and_and_16 - %4 = atomicrmw and ptr %a, i128 1 seq_cst -; CHECK: __sync_fetch_and_nand_16 - %5 = atomicrmw nand ptr %a, i128 1 seq_cst -; CHECK: __sync_fetch_and_or_16 - %6 = atomicrmw or ptr %a, i128 1 seq_cst -; CHECK: __sync_fetch_and_xor_16 - %7 = atomicrmw xor ptr %a, i128 1 seq_cst - ret void -} diff --git a/llvm/test/CodeGen/X86/nontemporal-3.ll b/llvm/test/CodeGen/X86/nontemporal-3.ll index a2d2c5ca4301..f9872b10097a 100644 --- a/llvm/test/CodeGen/X86/nontemporal-3.ll +++ b/llvm/test/CodeGen/X86/nontemporal-3.ll @@ -93,247 +93,66 @@ define void @test_zero_v4f64_align1(ptr %dst) nounwind { } define void @test_zero_v8f32_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v8f32_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v8f32_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorl %eax, %eax -; SSE4A-NEXT: movntiq %rax, 8(%rdi) -; SSE4A-NEXT: movntiq %rax, 24(%rdi) -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v8f32_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v8f32_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v8f32_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v8f32_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: retq store <8 x float> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v4i64_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v4i64_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v4i64_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v4i64_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v4i64_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v4i64_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v4i64_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: retq store <4 x i64> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v8i32_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v8i32_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v8i32_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v8i32_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v8i32_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v8i32_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v8i32_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: retq store <8 x i32> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v16i16_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v16i16_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v16i16_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v16i16_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v16i16_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v16i16_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v16i16_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: retq store <16 x i16> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v32i8_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v32i8_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v32i8_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v32i8_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v32i8_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v32i8_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v32i8_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: retq store <32 x i8> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } @@ -508,347 +327,86 @@ define void @test_zero_v8f64_align1(ptr %dst) nounwind { } define void @test_zero_v16f32_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v16f32_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: movntiq %rax, 40(%rdi) -; SSE2-NEXT: movntiq %rax, 32(%rdi) -; SSE2-NEXT: movntiq %rax, 56(%rdi) -; SSE2-NEXT: movntiq %rax, 48(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v16f32_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorl %eax, %eax -; SSE4A-NEXT: movntiq %rax, 8(%rdi) -; SSE4A-NEXT: movntiq %rax, 24(%rdi) -; SSE4A-NEXT: movntiq %rax, 40(%rdi) -; SSE4A-NEXT: movntiq %rax, 56(%rdi) -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 32(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 48(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v16f32_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: movntiq %rax, 40(%rdi) -; SSE41-NEXT: movntiq %rax, 32(%rdi) -; SSE41-NEXT: movntiq %rax, 56(%rdi) -; SSE41-NEXT: movntiq %rax, 48(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v16f32_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: movntiq %rax, 40(%rdi) -; AVX-NEXT: movntiq %rax, 32(%rdi) -; AVX-NEXT: movntiq %rax, 56(%rdi) -; AVX-NEXT: movntiq %rax, 48(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v16f32_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: movntiq %rax, 40(%rdi) -; AVX512-NEXT: movntiq %rax, 32(%rdi) -; AVX512-NEXT: movntiq %rax, 56(%rdi) -; AVX512-NEXT: movntiq %rax, 48(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v16f32_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: movntiq %rax, 40(%rdi) +; CHECK-NEXT: movntiq %rax, 32(%rdi) +; CHECK-NEXT: movntiq %rax, 56(%rdi) +; CHECK-NEXT: movntiq %rax, 48(%rdi) +; CHECK-NEXT: retq store <16 x float> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v8i64_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v8i64_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: movntiq %rax, 40(%rdi) -; SSE2-NEXT: movntiq %rax, 32(%rdi) -; SSE2-NEXT: movntiq %rax, 56(%rdi) -; SSE2-NEXT: movntiq %rax, 48(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v8i64_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 40(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 32(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 56(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 48(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v8i64_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: movntiq %rax, 40(%rdi) -; SSE41-NEXT: movntiq %rax, 32(%rdi) -; SSE41-NEXT: movntiq %rax, 56(%rdi) -; SSE41-NEXT: movntiq %rax, 48(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v8i64_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: movntiq %rax, 40(%rdi) -; AVX-NEXT: movntiq %rax, 32(%rdi) -; AVX-NEXT: movntiq %rax, 56(%rdi) -; AVX-NEXT: movntiq %rax, 48(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v8i64_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: movntiq %rax, 40(%rdi) -; AVX512-NEXT: movntiq %rax, 32(%rdi) -; AVX512-NEXT: movntiq %rax, 56(%rdi) -; AVX512-NEXT: movntiq %rax, 48(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v8i64_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: movntiq %rax, 40(%rdi) +; CHECK-NEXT: movntiq %rax, 32(%rdi) +; CHECK-NEXT: movntiq %rax, 56(%rdi) +; CHECK-NEXT: movntiq %rax, 48(%rdi) +; CHECK-NEXT: retq store <8 x i64> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v16i32_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v16i32_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: movntiq %rax, 40(%rdi) -; SSE2-NEXT: movntiq %rax, 32(%rdi) -; SSE2-NEXT: movntiq %rax, 56(%rdi) -; SSE2-NEXT: movntiq %rax, 48(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v16i32_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 40(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 32(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 56(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 48(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v16i32_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: movntiq %rax, 40(%rdi) -; SSE41-NEXT: movntiq %rax, 32(%rdi) -; SSE41-NEXT: movntiq %rax, 56(%rdi) -; SSE41-NEXT: movntiq %rax, 48(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v16i32_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: movntiq %rax, 40(%rdi) -; AVX-NEXT: movntiq %rax, 32(%rdi) -; AVX-NEXT: movntiq %rax, 56(%rdi) -; AVX-NEXT: movntiq %rax, 48(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v16i32_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: movntiq %rax, 40(%rdi) -; AVX512-NEXT: movntiq %rax, 32(%rdi) -; AVX512-NEXT: movntiq %rax, 56(%rdi) -; AVX512-NEXT: movntiq %rax, 48(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v16i32_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: movntiq %rax, 40(%rdi) +; CHECK-NEXT: movntiq %rax, 32(%rdi) +; CHECK-NEXT: movntiq %rax, 56(%rdi) +; CHECK-NEXT: movntiq %rax, 48(%rdi) +; CHECK-NEXT: retq store <16 x i32> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v32i16_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v32i16_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: movntiq %rax, 40(%rdi) -; SSE2-NEXT: movntiq %rax, 32(%rdi) -; SSE2-NEXT: movntiq %rax, 56(%rdi) -; SSE2-NEXT: movntiq %rax, 48(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v32i16_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 40(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 32(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 56(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 48(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v32i16_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: movntiq %rax, 40(%rdi) -; SSE41-NEXT: movntiq %rax, 32(%rdi) -; SSE41-NEXT: movntiq %rax, 56(%rdi) -; SSE41-NEXT: movntiq %rax, 48(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v32i16_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: movntiq %rax, 40(%rdi) -; AVX-NEXT: movntiq %rax, 32(%rdi) -; AVX-NEXT: movntiq %rax, 56(%rdi) -; AVX-NEXT: movntiq %rax, 48(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v32i16_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: movntiq %rax, 40(%rdi) -; AVX512-NEXT: movntiq %rax, 32(%rdi) -; AVX512-NEXT: movntiq %rax, 56(%rdi) -; AVX512-NEXT: movntiq %rax, 48(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v32i16_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: movntiq %rax, 40(%rdi) +; CHECK-NEXT: movntiq %rax, 32(%rdi) +; CHECK-NEXT: movntiq %rax, 56(%rdi) +; CHECK-NEXT: movntiq %rax, 48(%rdi) +; CHECK-NEXT: retq store <32 x i16> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } define void @test_zero_v64i8_align1(ptr %dst) nounwind { -; SSE2-LABEL: test_zero_v64i8_align1: -; SSE2: # %bb.0: -; SSE2-NEXT: xorl %eax, %eax -; SSE2-NEXT: movntiq %rax, 8(%rdi) -; SSE2-NEXT: movntiq %rax, (%rdi) -; SSE2-NEXT: movntiq %rax, 24(%rdi) -; SSE2-NEXT: movntiq %rax, 16(%rdi) -; SSE2-NEXT: movntiq %rax, 40(%rdi) -; SSE2-NEXT: movntiq %rax, 32(%rdi) -; SSE2-NEXT: movntiq %rax, 56(%rdi) -; SSE2-NEXT: movntiq %rax, 48(%rdi) -; SSE2-NEXT: retq -; -; SSE4A-LABEL: test_zero_v64i8_align1: -; SSE4A: # %bb.0: -; SSE4A-NEXT: xorps %xmm0, %xmm0 -; SSE4A-NEXT: movntsd %xmm0, 8(%rdi) -; SSE4A-NEXT: movntsd %xmm0, (%rdi) -; SSE4A-NEXT: movntsd %xmm0, 24(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 16(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 40(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 32(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 56(%rdi) -; SSE4A-NEXT: movntsd %xmm0, 48(%rdi) -; SSE4A-NEXT: retq -; -; SSE41-LABEL: test_zero_v64i8_align1: -; SSE41: # %bb.0: -; SSE41-NEXT: xorl %eax, %eax -; SSE41-NEXT: movntiq %rax, 8(%rdi) -; SSE41-NEXT: movntiq %rax, (%rdi) -; SSE41-NEXT: movntiq %rax, 24(%rdi) -; SSE41-NEXT: movntiq %rax, 16(%rdi) -; SSE41-NEXT: movntiq %rax, 40(%rdi) -; SSE41-NEXT: movntiq %rax, 32(%rdi) -; SSE41-NEXT: movntiq %rax, 56(%rdi) -; SSE41-NEXT: movntiq %rax, 48(%rdi) -; SSE41-NEXT: retq -; -; AVX-LABEL: test_zero_v64i8_align1: -; AVX: # %bb.0: -; AVX-NEXT: xorl %eax, %eax -; AVX-NEXT: movntiq %rax, 8(%rdi) -; AVX-NEXT: movntiq %rax, (%rdi) -; AVX-NEXT: movntiq %rax, 24(%rdi) -; AVX-NEXT: movntiq %rax, 16(%rdi) -; AVX-NEXT: movntiq %rax, 40(%rdi) -; AVX-NEXT: movntiq %rax, 32(%rdi) -; AVX-NEXT: movntiq %rax, 56(%rdi) -; AVX-NEXT: movntiq %rax, 48(%rdi) -; AVX-NEXT: retq -; -; AVX512-LABEL: test_zero_v64i8_align1: -; AVX512: # %bb.0: -; AVX512-NEXT: xorl %eax, %eax -; AVX512-NEXT: movntiq %rax, 8(%rdi) -; AVX512-NEXT: movntiq %rax, (%rdi) -; AVX512-NEXT: movntiq %rax, 24(%rdi) -; AVX512-NEXT: movntiq %rax, 16(%rdi) -; AVX512-NEXT: movntiq %rax, 40(%rdi) -; AVX512-NEXT: movntiq %rax, 32(%rdi) -; AVX512-NEXT: movntiq %rax, 56(%rdi) -; AVX512-NEXT: movntiq %rax, 48(%rdi) -; AVX512-NEXT: retq +; CHECK-LABEL: test_zero_v64i8_align1: +; CHECK: # %bb.0: +; CHECK-NEXT: xorl %eax, %eax +; CHECK-NEXT: movntiq %rax, 8(%rdi) +; CHECK-NEXT: movntiq %rax, (%rdi) +; CHECK-NEXT: movntiq %rax, 24(%rdi) +; CHECK-NEXT: movntiq %rax, 16(%rdi) +; CHECK-NEXT: movntiq %rax, 40(%rdi) +; CHECK-NEXT: movntiq %rax, 32(%rdi) +; CHECK-NEXT: movntiq %rax, 56(%rdi) +; CHECK-NEXT: movntiq %rax, 48(%rdi) +; CHECK-NEXT: retq store <64 x i8> zeroinitializer, ptr %dst, align 1, !nontemporal !1 ret void } @@ -1214,3 +772,7 @@ define void @test_zero_v64i8_align32(ptr %dst) nounwind { } !1 = !{i32 1} +;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: +; SSE2: {{.*}} +; SSE41: {{.*}} +; SSE4A: {{.*}} diff --git a/llvm/test/CodeGen/X86/pr38738.ll b/llvm/test/CodeGen/X86/pr38738.ll index 753b7ce33d2b..205849e7d05d 100644 --- a/llvm/test/CodeGen/X86/pr38738.ll +++ b/llvm/test/CodeGen/X86/pr38738.ll @@ -130,22 +130,15 @@ define void @tryset(ptr nocapture %x) { ; X86SSE2-LABEL: tryset: ; X86SSE2: # %bb.0: ; X86SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86SSE2-NEXT: movl $0, 4(%eax) -; X86SSE2-NEXT: movl $0, (%eax) -; X86SSE2-NEXT: movl $0, 12(%eax) -; X86SSE2-NEXT: movl $0, 8(%eax) -; X86SSE2-NEXT: movl $0, 20(%eax) -; X86SSE2-NEXT: movl $0, 16(%eax) -; X86SSE2-NEXT: movl $0, 28(%eax) -; X86SSE2-NEXT: movl $0, 24(%eax) -; X86SSE2-NEXT: movl $0, 36(%eax) -; X86SSE2-NEXT: movl $0, 32(%eax) -; X86SSE2-NEXT: movl $0, 44(%eax) -; X86SSE2-NEXT: movl $0, 40(%eax) -; X86SSE2-NEXT: movl $0, 52(%eax) -; X86SSE2-NEXT: movl $0, 48(%eax) -; X86SSE2-NEXT: movl $0, 60(%eax) -; X86SSE2-NEXT: movl $0, 56(%eax) +; X86SSE2-NEXT: xorps %xmm0, %xmm0 +; X86SSE2-NEXT: movsd %xmm0, 56(%eax) +; X86SSE2-NEXT: movsd %xmm0, 48(%eax) +; X86SSE2-NEXT: movsd %xmm0, 40(%eax) +; X86SSE2-NEXT: movsd %xmm0, 32(%eax) +; X86SSE2-NEXT: movsd %xmm0, 24(%eax) +; X86SSE2-NEXT: movsd %xmm0, 16(%eax) +; X86SSE2-NEXT: movsd %xmm0, 8(%eax) +; X86SSE2-NEXT: movsd %xmm0, (%eax) ; X86SSE2-NEXT: retl ; ; X64AVX-LABEL: tryset: diff --git a/llvm/test/CodeGen/X86/pr41619.ll b/llvm/test/CodeGen/X86/pr41619.ll index 7d1d139a38a5..88dcd7798f0c 100644 --- a/llvm/test/CodeGen/X86/pr41619.ll +++ b/llvm/test/CodeGen/X86/pr41619.ll @@ -7,10 +7,9 @@ define void @foo(double %arg) { ; CHECK: ## %bb.0: ## %bb ; CHECK-NEXT: vmovq %xmm0, %rax ; CHECK-NEXT: vmovd %eax, %xmm0 -; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1 ; CHECK-NEXT: vmovq %xmm0, %rax ; CHECK-NEXT: movl %eax, (%rax) -; CHECK-NEXT: vmovlps %xmm1, (%rax) +; CHECK-NEXT: movq $0, (%rax) ; CHECK-NEXT: retq bb: %tmp = bitcast double %arg to i64 diff --git a/llvm/test/CodeGen/X86/pr74736.ll b/llvm/test/CodeGen/X86/pr74736.ll new file mode 100644 index 000000000000..3dfdbf102c95 --- /dev/null +++ b/llvm/test/CodeGen/X86/pr74736.ll @@ -0,0 +1,68 @@ +; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 4 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX + +define void @main(<16 x i32> %0, i32 %1) { +; SSE-LABEL: main: +; SSE: # %bb.0: # %entry +; SSE-NEXT: movd %edi, %xmm4 +; SSE-NEXT: movaps {{.*#+}} xmm0 = [1,0,0,0] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm4[1,0] +; SSE-NEXT: paddd %xmm0, %xmm0 +; SSE-NEXT: paddd %xmm1, %xmm1 +; SSE-NEXT: paddd %xmm3, %xmm3 +; SSE-NEXT: paddd %xmm2, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,1,1,3] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,0],xmm1[1,0] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[1,3] +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,0],xmm2[1,0] +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm2[1,3] +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,0],xmm3[1,0] +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[1,3] +; SSE-NEXT: xorps %xmm2, %xmm0 +; SSE-NEXT: xorps %xmm4, %xmm1 +; SSE-NEXT: xorps %xmm0, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3] +; SSE-NEXT: pxor %xmm1, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] +; SSE-NEXT: pxor %xmm0, %xmm1 +; SSE-NEXT: movd %xmm1, 0 +; SSE-NEXT: retq +; +; AVX-LABEL: main: +; AVX: # %bb.0: # %entry +; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3] +; AVX-NEXT: movl $1, %eax +; AVX-NEXT: vpinsrd $1, %eax, %xmm2, %xmm2 +; AVX-NEXT: vpinsrd $3, %edi, %xmm2, %xmm2 +; AVX-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm0[4,5,6,7] +; AVX-NEXT: vpaddd %ymm0, %ymm0, %ymm0 +; AVX-NEXT: vpaddd %ymm1, %ymm1, %ymm1 +; AVX-NEXT: vmovdqa {{.*#+}} ymm2 = [0,1,1,3,3,5,5,7] +; AVX-NEXT: vpermd %ymm0, %ymm2, %ymm2 +; AVX-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm1[0,1] +; AVX-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[3,3,3,3,7,7,7,7] +; AVX-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,1,1,3,4,5,5,7] +; AVX-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7] +; AVX-NEXT: vpxor %ymm0, %ymm2, %ymm0 +; AVX-NEXT: vextracti128 $1, %ymm0, %xmm1 +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3] +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] +; AVX-NEXT: vpxor %xmm1, %xmm0, %xmm0 +; AVX-NEXT: vmovd %xmm0, 0 +; AVX-NEXT: vzeroupper +; AVX-NEXT: retq +entry: + %2 = insertelement <16 x i32> %0, i32 1, i64 1 + %3 = insertelement <16 x i32> %2, i32 %1, i64 3 + %4 = insertelement <16 x i32> %3, i32 0, i64 0 + %5 = shl <16 x i32> %4, + %6 = shufflevector <16 x i32> %5, <16 x i32> zeroinitializer, <16 x i32> + %7 = tail call i32 @llvm.vector.reduce.xor.v16i32(<16 x i32> %6) + store i32 %7, ptr null, align 4 + ret void +} +declare i32 @llvm.vector.reduce.xor.v16i32(<16 x i32>) diff --git a/llvm/test/CodeGen/X86/slow-unaligned-mem.ll b/llvm/test/CodeGen/X86/slow-unaligned-mem.ll index 85afa83e3f08..d74d195439bd 100644 --- a/llvm/test/CodeGen/X86/slow-unaligned-mem.ll +++ b/llvm/test/CodeGen/X86/slow-unaligned-mem.ll @@ -1,16 +1,16 @@ ; Intel chips with slow unaligned memory accesses -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium3 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium3m 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium-m 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium4 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium4m 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=yonah 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=prescott 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=nocona 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=core2 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=penryn 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=bonnell 2>&1 | FileCheck %s --check-prefixes=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium3 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SCALAR +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium3m 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SCALAR +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium-m 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium4 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=pentium4m 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=yonah 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=prescott 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=nocona 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=core2 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=penryn 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=bonnell 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE ; Intel chips with fast unaligned memory accesses @@ -26,15 +26,15 @@ ; AMD chips with slow unaligned memory accesses -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-4 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-xp 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=k8 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=opteron 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon64 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-fx 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=k8-sse3 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=opteron-sse3 2>&1 | FileCheck %s --check-prefixes=SLOW -; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon64-sse3 2>&1 | FileCheck %s --check-prefixes=SLOW +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-4 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SCALAR +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-xp 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SCALAR +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=k8 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=opteron 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon64 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon-fx 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=k8-sse3 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=opteron-sse3 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE +; RUN: llc < %s -mtriple=i386-unknown-unknown -mcpu=athlon64-sse3 2>&1 | FileCheck %s --check-prefixes=SLOW,SLOW-SSE ; AMD chips with fast unaligned memory accesses @@ -67,26 +67,40 @@ ; SLOW-NOT: not a recognized processor ; FAST-NOT: not a recognized processor define void @store_zeros(ptr %a) { -; SLOW-LABEL: store_zeros: -; SLOW: # %bb.0: -; SLOW-NEXT: movl {{[0-9]+}}(%esp), %eax -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NEXT: movl $0 -; SLOW-NOT: movl +; SLOW-SCALAR-LABEL: store_zeros: +; SLOW-SCALAR: # %bb.0: +; SLOW-SCALAR-NEXT: movl {{[0-9]+}}(%esp), %eax +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NEXT: movl $0 +; SLOW-SCALAR-NOT: movl +; +; SLOW-SSE-LABEL: store_zeros: +; SLOW-SSE: # %bb.0: +; SLOW-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax +; SLOW-SSE-NEXT: xorps %xmm0, %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NEXT: movsd %xmm0 +; SLOW-SSE-NOT: movsd ; ; FAST-SSE-LABEL: store_zeros: ; FAST-SSE: # %bb.0: diff --git a/llvm/test/CodeGen/X86/var-permute-256.ll b/llvm/test/CodeGen/X86/var-permute-256.ll index 6c07c4ca523f..853d8a278960 100644 --- a/llvm/test/CodeGen/X86/var-permute-256.ll +++ b/llvm/test/CodeGen/X86/var-permute-256.ll @@ -25,18 +25,20 @@ define <4 x i64> @var_shuffle_v4i64(<4 x i64> %v, <4 x i64> %indices) nounwind { ; ; AVX1-LABEL: var_shuffle_v4i64: ; AVX1: # %bb.0: -; AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3] -; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm3 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpaddq %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = [2,2] +; AVX1-NEXT: # xmm3 = mem[0,0] +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm4 ; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm4 -; AVX1-NEXT: vpermilpd %ymm4, %ymm2, %ymm2 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3 +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[2,3,2,3] +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpermilpd %ymm1, %ymm4, %ymm2 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX1-NEXT: vpermilpd %ymm4, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 -; AVX1-NEXT: vblendvpd %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm3, %ymm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_shuffle_v4i64: @@ -88,15 +90,16 @@ define <8 x i32> @var_shuffle_v8i32(<8 x i32> %v, <8 x i32> %indices) nounwind { ; ; AVX1-LABEL: var_shuffle_v8i32: ; AVX1: # %bb.0: -; AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3] -; AVX1-NEXT: vpermilps %ymm1, %ymm2, %ymm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [3,3,3,3] +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3] +; AVX1-NEXT: vpermilps %ymm1, %ymm3, %ymm3 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 -; AVX1-NEXT: vblendvps %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvps %ymm2, %ymm3, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; INT256-LABEL: var_shuffle_v8i32: @@ -445,18 +448,20 @@ define <4 x double> @var_shuffle_v4f64(<4 x double> %v, <4 x i64> %indices) noun ; ; AVX1-LABEL: var_shuffle_v4f64: ; AVX1: # %bb.0: -; AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3] -; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm3 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpaddq %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = [2,2] +; AVX1-NEXT: # xmm3 = mem[0,0] +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm4 ; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm4 -; AVX1-NEXT: vpermilpd %ymm4, %ymm2, %ymm2 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3 +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[2,3,2,3] +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpermilpd %ymm1, %ymm4, %ymm2 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX1-NEXT: vpermilpd %ymm4, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3, %xmm3 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 -; AVX1-NEXT: vblendvpd %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm3, %ymm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_shuffle_v4f64: @@ -508,15 +513,16 @@ define <8 x float> @var_shuffle_v8f32(<8 x float> %v, <8 x i32> %indices) nounwi ; ; AVX1-LABEL: var_shuffle_v8f32: ; AVX1: # %bb.0: -; AVX1-NEXT: vperm2f128 {{.*#+}} ymm2 = ymm0[2,3,2,3] -; AVX1-NEXT: vpermilps %ymm1, %ymm2, %ymm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [3,3,3,3] +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3] +; AVX1-NEXT: vpermilps %ymm1, %ymm3, %ymm3 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 -; AVX1-NEXT: vblendvps %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvps %ymm2, %ymm3, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; INT256-LABEL: var_shuffle_v8f32: @@ -569,17 +575,19 @@ define <4 x i64> @var_shuffle_v4i64_from_v2i64(<2 x i64> %v, <4 x i64> %indices) ; AVX1-LABEL: var_shuffle_v4i64_from_v2i64: ; AVX1: # %bb.0: ; AVX1-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0 -; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm2 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpaddq %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = [2,2] +; AVX1-NEXT: # xmm3 = mem[0,0] +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm4 ; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm3 -; AVX1-NEXT: vpermilpd %ymm3, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 -; AVX1-NEXT: vpermilpd %ymm3, %ymm0, %ymm2 -; AVX1-NEXT: vblendvpd %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm1 +; AVX1-NEXT: vblendvpd %ymm3, %ymm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_shuffle_v4i64_from_v2i64: @@ -633,14 +641,15 @@ define <8 x i32> @var_shuffle_v8i32_from_v4i32(<4 x i32> %v, <8 x i32> %indices) ; AVX1-LABEL: var_shuffle_v8i32_from_v4i32: ; AVX1: # %bb.0: # %entry ; AVX1-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0 -; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [3,3,3,3] +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm3 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 -; AVX1-NEXT: vblendvps %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvps %ymm2, %ymm3, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; INT256-LABEL: var_shuffle_v8i32_from_v4i32: @@ -990,17 +999,19 @@ define <4 x double> @var_shuffle_v4f64_from_v2f64(<2 x double> %v, <4 x i64> %in ; AVX1-LABEL: var_shuffle_v4f64_from_v2f64: ; AVX1: # %bb.0: ; AVX1-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0 -; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm2 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpaddq %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = [2,2] +; AVX1-NEXT: # xmm3 = mem[0,0] +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm4 ; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm3 -; AVX1-NEXT: vpermilpd %ymm3, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2, %xmm2 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 -; AVX1-NEXT: vpermilpd %ymm3, %ymm0, %ymm2 -; AVX1-NEXT: vblendvpd %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm1 +; AVX1-NEXT: vblendvpd %ymm3, %ymm1, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_shuffle_v4f64_from_v2f64: @@ -1054,14 +1065,15 @@ define <8 x float> @var_shuffle_v8f32_from_v4f32(<4 x float> %v, <8 x i32> %indi ; AVX1-LABEL: var_shuffle_v8f32_from_v4f32: ; AVX1: # %bb.0: # %entry ; AVX1-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0 -; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm2 +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [3,3,3,3] +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpcmpgtd %xmm3, %xmm1, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm3, %ymm2 +; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm3 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 ; AVX1-NEXT: vpermilps %ymm1, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3 -; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX1-NEXT: vpcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm3, %ymm1 -; AVX1-NEXT: vblendvps %ymm1, %ymm2, %ymm0, %ymm0 +; AVX1-NEXT: vblendvps %ymm2, %ymm3, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; INT256-LABEL: var_shuffle_v8f32_from_v4f32: @@ -1271,20 +1283,22 @@ define <4 x i64> @var_shuffle_v4i64_with_v16i8_indices(<4 x i64> %v, <16 x i8> % ; ; AVX1-LABEL: var_shuffle_v4i64_with_v16i8_indices: ; AVX1: # %bb.0: -; AVX1-NEXT: vpsrld $16, %xmm1, %xmm2 -; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpsrld $16, %xmm1, %xmm1 ; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero -; AVX1-NEXT: vperm2f128 {{.*#+}} ymm3 = ymm0[2,3,2,3] ; AVX1-NEXT: vpaddq %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vmovddup {{.*#+}} xmm3 = [2,2] +; AVX1-NEXT: # xmm3 = mem[0,0] +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm1, %xmm4 ; AVX1-NEXT: vpaddq %xmm2, %xmm2, %xmm2 -; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm4 -; AVX1-NEXT: vpermilpd %ymm4, %ymm3, %ymm3 +; AVX1-NEXT: vpcmpgtq %xmm3, %xmm2, %xmm3 +; AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm3, %ymm3 +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm4 = ymm0[2,3,2,3] +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1 +; AVX1-NEXT: vpermilpd %ymm1, %ymm4, %ymm2 ; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX1-NEXT: vpermilpd %ymm4, %ymm0, %ymm0 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 -; AVX1-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm2, %xmm2 -; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 -; AVX1-NEXT: vblendvpd %ymm1, %ymm3, %ymm0, %ymm0 +; AVX1-NEXT: vpermilpd %ymm1, %ymm0, %ymm0 +; AVX1-NEXT: vblendvpd %ymm3, %ymm2, %ymm0, %ymm0 ; AVX1-NEXT: retq ; ; AVX2-LABEL: var_shuffle_v4i64_with_v16i8_indices: diff --git a/llvm/test/CodeGen/X86/vec_fcopysign.ll b/llvm/test/CodeGen/X86/vec_fcopysign.ll index 457210732396..b34b02c90796 100644 --- a/llvm/test/CodeGen/X86/vec_fcopysign.ll +++ b/llvm/test/CodeGen/X86/vec_fcopysign.ll @@ -159,8 +159,8 @@ define <8 x half> @fcopysign_v8f16(ptr %p0, ptr %p1) nounwind { ; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-AVX512-NEXT: vmovdqa (%ecx), %xmm1 -; X86-AVX512-NEXT: vpbroadcastw {{.*#+}} xmm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; X86-AVX512-NEXT: vpternlogq $202, (%eax), %xmm1, %xmm0 +; X86-AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [2147450879,2147450879,2147450879,2147450879] +; X86-AVX512-NEXT: vpternlogd $202, (%eax), %xmm1, %xmm0 ; X86-AVX512-NEXT: retl ; ; X64-SSE-LABEL: fcopysign_v8f16: @@ -193,8 +193,8 @@ define <8 x half> @fcopysign_v8f16(ptr %p0, ptr %p1) nounwind { ; X64-AVX512-LABEL: fcopysign_v8f16: ; X64-AVX512: # %bb.0: ; X64-AVX512-NEXT: vmovdqa (%rdi), %xmm1 -; X64-AVX512-NEXT: vpbroadcastq {{.*#+}} xmm0 = [9223231297218904063,9223231297218904063] -; X64-AVX512-NEXT: vpternlogq $202, (%rsi), %xmm1, %xmm0 +; X64-AVX512-NEXT: vpbroadcastd {{.*#+}} xmm0 = [2147450879,2147450879,2147450879,2147450879] +; X64-AVX512-NEXT: vpternlogd $202, (%rsi), %xmm1, %xmm0 ; X64-AVX512-NEXT: retq %a0 = load <8 x half>, ptr %p0, align 16 %a1 = load <8 x half>, ptr %p1, align 16 @@ -405,8 +405,8 @@ define <16 x half> @fcopysign_v16f16(ptr %p0, ptr %p1) nounwind { ; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax ; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %ecx ; X86-AVX512-NEXT: vmovdqu (%ecx), %ymm1 -; X86-AVX512-NEXT: vpbroadcastw {{.*#+}} ymm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; X86-AVX512-NEXT: vpternlogq $202, (%eax), %ymm1, %ymm0 +; X86-AVX512-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879] +; X86-AVX512-NEXT: vpternlogd $202, (%eax), %ymm1, %ymm0 ; X86-AVX512-NEXT: retl ; ; X64-SSE-LABEL: fcopysign_v16f16: @@ -444,8 +444,8 @@ define <16 x half> @fcopysign_v16f16(ptr %p0, ptr %p1) nounwind { ; X64-AVX512-LABEL: fcopysign_v16f16: ; X64-AVX512: # %bb.0: ; X64-AVX512-NEXT: vmovdqu (%rdi), %ymm1 -; X64-AVX512-NEXT: vpbroadcastq {{.*#+}} ymm0 = [9223231297218904063,9223231297218904063,9223231297218904063,9223231297218904063] -; X64-AVX512-NEXT: vpternlogq $202, (%rsi), %ymm1, %ymm0 +; X64-AVX512-NEXT: vpbroadcastd {{.*#+}} ymm0 = [2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879] +; X64-AVX512-NEXT: vpternlogd $202, (%rsi), %ymm1, %ymm0 ; X64-AVX512-NEXT: retq %a0 = load <16 x half>, ptr %p0, align 16 %a1 = load <16 x half>, ptr %p1, align 16 @@ -691,34 +691,14 @@ define <32 x half> @fcopysign_v32f16(ptr %p0, ptr %p1) nounwind { ; X86-AVX2-NEXT: vpor %ymm2, %ymm1, %ymm1 ; X86-AVX2-NEXT: retl ; -; X86-AVX512VL-LABEL: fcopysign_v32f16: -; X86-AVX512VL: # %bb.0: -; X86-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-AVX512VL-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-AVX512VL-NEXT: vmovdqu64 (%ecx), %zmm1 -; X86-AVX512VL-NEXT: vpbroadcastw {{.*#+}} ymm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; X86-AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 -; X86-AVX512VL-NEXT: vpternlogq $202, (%eax), %zmm1, %zmm0 -; X86-AVX512VL-NEXT: retl -; -; X86-AVX512FP16-LABEL: fcopysign_v32f16: -; X86-AVX512FP16: # %bb.0: -; X86-AVX512FP16-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-AVX512FP16-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-AVX512FP16-NEXT: vmovdqu64 (%ecx), %zmm1 -; X86-AVX512FP16-NEXT: vpbroadcastw {{.*#+}} zmm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; X86-AVX512FP16-NEXT: vpternlogq $202, (%eax), %zmm1, %zmm0 -; X86-AVX512FP16-NEXT: retl -; -; X86-AVX512VLDQ-LABEL: fcopysign_v32f16: -; X86-AVX512VLDQ: # %bb.0: -; X86-AVX512VLDQ-NEXT: movl {{[0-9]+}}(%esp), %eax -; X86-AVX512VLDQ-NEXT: movl {{[0-9]+}}(%esp), %ecx -; X86-AVX512VLDQ-NEXT: vmovdqu64 (%ecx), %zmm1 -; X86-AVX512VLDQ-NEXT: vpbroadcastw {{.*#+}} ymm0 = [NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN,NaN] -; X86-AVX512VLDQ-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 -; X86-AVX512VLDQ-NEXT: vpternlogq $202, (%eax), %zmm1, %zmm0 -; X86-AVX512VLDQ-NEXT: retl +; X86-AVX512-LABEL: fcopysign_v32f16: +; X86-AVX512: # %bb.0: +; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %eax +; X86-AVX512-NEXT: movl {{[0-9]+}}(%esp), %ecx +; X86-AVX512-NEXT: vmovdqu64 (%ecx), %zmm1 +; X86-AVX512-NEXT: vpbroadcastd {{.*#+}} zmm0 = [2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879] +; X86-AVX512-NEXT: vpternlogd $202, (%eax), %zmm1, %zmm0 +; X86-AVX512-NEXT: retl ; ; X64-SSE-LABEL: fcopysign_v32f16: ; X64-SSE: # %bb.0: @@ -769,8 +749,8 @@ define <32 x half> @fcopysign_v32f16(ptr %p0, ptr %p1) nounwind { ; X64-AVX512-LABEL: fcopysign_v32f16: ; X64-AVX512: # %bb.0: ; X64-AVX512-NEXT: vmovdqu64 (%rdi), %zmm1 -; X64-AVX512-NEXT: vpbroadcastq {{.*#+}} zmm0 = [9223231297218904063,9223231297218904063,9223231297218904063,9223231297218904063,9223231297218904063,9223231297218904063,9223231297218904063,9223231297218904063] -; X64-AVX512-NEXT: vpternlogq $202, (%rsi), %zmm1, %zmm0 +; X64-AVX512-NEXT: vpbroadcastd {{.*#+}} zmm0 = [2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879,2147450879] +; X64-AVX512-NEXT: vpternlogd $202, (%rsi), %zmm1, %zmm0 ; X64-AVX512-NEXT: retq %a0 = load <32 x half>, ptr %p0, align 16 %a1 = load <32 x half>, ptr %p1, align 16 @@ -786,3 +766,6 @@ declare <32 x half> @llvm.copysign.v32f16(<32 x half>, <32 x half>) ; X64-AVX512VLDQ: {{.*}} ; X86: {{.*}} ; X86-AVX: {{.*}} +; X86-AVX512FP16: {{.*}} +; X86-AVX512VL: {{.*}} +; X86-AVX512VLDQ: {{.*}} diff --git a/llvm/test/CodeGen/X86/vec_zero_cse.ll b/llvm/test/CodeGen/X86/vec_zero_cse.ll index 21da191d8d69..d4357aeb2e1d 100644 --- a/llvm/test/CodeGen/X86/vec_zero_cse.ll +++ b/llvm/test/CodeGen/X86/vec_zero_cse.ll @@ -16,7 +16,7 @@ define void @test1() { ; X86-NEXT: movl $0, M1+4 ; X86-NEXT: movl $0, M1 ; X86-NEXT: xorps %xmm0, %xmm0 -; X86-NEXT: movlps %xmm0, M2 +; X86-NEXT: movsd %xmm0, M2 ; X86-NEXT: retl ; ; X64-LABEL: test1: diff --git a/llvm/test/CodeGen/X86/vector-fshl-128.ll b/llvm/test/CodeGen/X86/vector-fshl-128.ll index 2d0e92a54846..4f100cd3e053 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-128.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-128.ll @@ -2409,7 +2409,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512F-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq @@ -2418,14 +2418,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VL-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v16i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512BW-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq @@ -2434,7 +2434,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VBMI2-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512VBMI2-NEXT: vzeroupper ; AVX512VBMI2-NEXT: retq @@ -2443,14 +2443,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VLBW-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v16i8: diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll index 43d426c326da..05be4e1ee928 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-256.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll @@ -2303,7 +2303,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512F-NEXT: retq ; @@ -2311,14 +2311,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v32i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512BW-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; @@ -2326,7 +2326,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VBMI2-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512VBMI2-NEXT: retq ; @@ -2334,14 +2334,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VLBW-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; AVX512VLBW-NEXT: retq ; ; AVX10-LABEL: splatconstant_funnnel_v32i8: ; AVX10: # %bb.0: ; AVX10-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX10-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX10-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; AVX10-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; AVX10-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8: diff --git a/llvm/test/CodeGen/X86/vector-fshl-512.ll b/llvm/test/CodeGen/X86/vector-fshl-512.ll index c246aaf61dbc..0724d879f557 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-512.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-512.ll @@ -1124,7 +1124,7 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi ; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm1 ; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm1 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: splatconstant_funnnel_v64i8: @@ -1137,35 +1137,35 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi ; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm1 ; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm1 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v64i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512BW-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512BW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512VBMI2-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VBMI2-NEXT: retq ; ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512VLBW-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> ) ret <64 x i8> %res diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll index 37d4f3b3dff5..9ddd171b4db6 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-rot-128.ll @@ -1851,7 +1851,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512F-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq @@ -1860,14 +1860,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v16i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512BW-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq @@ -1876,14 +1876,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VLBW-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512VBMI2-NEXT: vzeroupper ; AVX512VBMI2-NEXT: retq @@ -1892,7 +1892,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v16i8: diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll index 8762072f3e8f..58719e6bd8e0 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll @@ -447,12 +447,12 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: vpsllw $2, %ymm0, %ymm2 ; AVX512F-NEXT: vpsrlw $6, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm2 @@ -467,12 +467,12 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsllw $2, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $6, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $7, %ymm0, %ymm2 @@ -1652,7 +1652,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512F-NEXT: retq ; @@ -1660,14 +1660,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v32i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512BW-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; @@ -1675,14 +1675,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VLBW-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512VBMI2-NEXT: retq ; @@ -1690,7 +1690,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8: diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll index 0a473dd1ed82..edfa56a70d59 100644 --- a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll +++ b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll @@ -131,15 +131,15 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind { ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm3 ; AVX512F-NEXT: vpsllw $4, %ymm2, %ymm4 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm5 = [17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520] -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm5, %zmm4 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm5 = [4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160] +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm5, %zmm4 ; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3 ; AVX512F-NEXT: vpsllw $5, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512F-NEXT: vpsrlw $6, %ymm2, %ymm4 ; AVX512F-NEXT: vpsllw $2, %ymm2, %ymm6 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm7 = [18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596] -; AVX512F-NEXT: vpternlogq $226, %zmm4, %zmm7, %zmm6 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm7 = [4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268] +; AVX512F-NEXT: vpternlogd $226, %zmm4, %zmm7, %zmm6 ; AVX512F-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512F-NEXT: vpsrlw $7, %ymm2, %ymm4 @@ -151,12 +151,12 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind { ; AVX512F-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm3 ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm5, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm5, %zmm4 ; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $6, %ymm0, %ymm3 ; AVX512F-NEXT: vpsllw $2, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm7, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm7, %zmm4 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm3 @@ -173,15 +173,15 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind { ; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm3 ; AVX512VL-NEXT: vpsllw $4, %ymm2, %ymm4 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm5 = [17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520] -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm5, %ymm4 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm5 = [4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160] +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm5, %ymm4 ; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm3 ; AVX512VL-NEXT: vpsllw $5, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsrlw $6, %ymm2, %ymm4 ; AVX512VL-NEXT: vpsllw $2, %ymm2, %ymm6 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm7 = [18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596] -; AVX512VL-NEXT: vpternlogq $226, %ymm4, %ymm7, %ymm6 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm7 = [4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268] +; AVX512VL-NEXT: vpternlogd $226, %ymm4, %ymm7, %ymm6 ; AVX512VL-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsrlw $7, %ymm2, %ymm4 @@ -192,12 +192,12 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind { ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm5, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm5, %ymm4 ; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $6, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsllw $2, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm7, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm7, %ymm4 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $7, %ymm0, %ymm3 @@ -721,7 +721,7 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind { ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: splatconstant_funnnel_v64i8: @@ -733,35 +733,35 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind { ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v64i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: retq ; ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshl.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> ) ret <64 x i8> %res diff --git a/llvm/test/CodeGen/X86/vector-fshr-128.ll b/llvm/test/CodeGen/X86/vector-fshr-128.ll index 809735a88f20..eb2df0dcda98 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-128.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-128.ll @@ -2403,7 +2403,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512F-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq @@ -2412,14 +2412,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VL-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v16i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512BW-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq @@ -2428,7 +2428,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VBMI2-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512VBMI2-NEXT: vzeroupper ; AVX512VBMI2-NEXT: retq @@ -2437,14 +2437,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x, <16 x i8> %y) nounwi ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VLBW-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v16i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %xmm0, %xmm2 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %xmm1, %xmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm2, %xmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm2, %xmm0 ; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v16i8: diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll index fa41a10adb5e..1a6ecea59656 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-256.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll @@ -2104,7 +2104,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512F-NEXT: retq ; @@ -2112,14 +2112,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v32i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512BW-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; @@ -2127,7 +2127,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VBMI2-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512VBMI2-NEXT: retq ; @@ -2135,14 +2135,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x, <32 x i8> %y) nounwi ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VLBW-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; AVX512VLBW-NEXT: retq ; ; AVX10-LABEL: splatconstant_funnnel_v32i8: ; AVX10: # %bb.0: ; AVX10-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX10-NEXT: vpsrlw $4, %ymm1, %ymm0 -; AVX10-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm0 +; AVX10-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm0 ; AVX10-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8: diff --git a/llvm/test/CodeGen/X86/vector-fshr-512.ll b/llvm/test/CodeGen/X86/vector-fshr-512.ll index b1fee9d1b0b3..1c6646152ab1 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-512.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-512.ll @@ -1180,7 +1180,7 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi ; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm1 ; AVX512F-NEXT: vpsrlw $4, %ymm1, %ymm1 ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: splatconstant_funnnel_v64i8: @@ -1193,35 +1193,35 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwi ; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm1 ; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm1 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v64i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512BW-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512BW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512VBMI2-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VBMI2-NEXT: retq ; ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512VLBW-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm2 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm1, %zmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm0 ; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> ) ret <64 x i8> %res diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll index 9c5fe49e7d0c..402eb73e1810 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-rot-128.ll @@ -1915,7 +1915,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512F-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512F-NEXT: vzeroupper ; AVX512F-NEXT: retq @@ -1924,14 +1924,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v16i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512BW-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512BW-NEXT: vzeroupper ; AVX512BW-NEXT: retq @@ -1940,14 +1940,14 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VLBW-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v16i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512VBMI2-NEXT: vzeroupper ; AVX512VBMI2-NEXT: retq @@ -1956,7 +1956,7 @@ define <16 x i8> @splatconstant_funnnel_v16i8(<16 x i8> %x) nounwind { ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VLVBMI2-NEXT: retq ; ; XOP-LABEL: splatconstant_funnnel_v16i8: diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll index e4867ba09696..bb311468ce91 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll @@ -473,17 +473,17 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm2 ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $2, %ymm0, %ymm2 ; AVX512F-NEXT: vpsllw $6, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $1, %ymm0, %ymm2 ; AVX512F-NEXT: vpsllw $7, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: retq @@ -492,17 +492,17 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $2, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsllw $6, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $1, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsllw $7, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: retq @@ -1703,7 +1703,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512F-NEXT: retq ; @@ -1711,14 +1711,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v32i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512BW-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512BW-NEXT: retq ; @@ -1726,14 +1726,14 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VLBW-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v32i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512VBMI2-NEXT: retq ; @@ -1741,7 +1741,7 @@ define <32 x i8> @splatconstant_funnnel_v32i8(<32 x i8> %x) nounwind { ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VLVBMI2-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_funnnel_v32i8: diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll index 3c17bf2f6b9a..4364c047dfde 100644 --- a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll +++ b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll @@ -131,36 +131,36 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind { ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2 ; AVX512F-NEXT: vpsllw $4, %ymm2, %ymm3 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm4 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm5 = [1085102592571150095,1085102592571150095,1085102592571150095,1085102592571150095,1085102592571150095,1085102592571150095,1085102592571150095,1085102592571150095] -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm5, %zmm4 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm5 = [252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135] +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm5, %zmm4 ; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3 ; AVX512F-NEXT: vpsllw $5, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512F-NEXT: vpsllw $6, %ymm2, %ymm4 ; AVX512F-NEXT: vpsrlw $2, %ymm2, %ymm6 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm7 = [4557430888798830399,4557430888798830399,4557430888798830399,4557430888798830399,4557430888798830399,4557430888798830399,4557430888798830399,4557430888798830399] -; AVX512F-NEXT: vpternlogq $226, %zmm4, %zmm7, %zmm6 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm7 = [1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567] +; AVX512F-NEXT: vpternlogd $226, %zmm4, %zmm7, %zmm6 ; AVX512F-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512F-NEXT: vpsllw $7, %ymm2, %ymm4 ; AVX512F-NEXT: vpsrlw $1, %ymm2, %ymm6 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm8 = [9187201950435737471,9187201950435737471,9187201950435737471,9187201950435737471,9187201950435737471,9187201950435737471,9187201950435737471,9187201950435737471] -; AVX512F-NEXT: vpternlogq $226, %zmm4, %zmm8, %zmm6 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm8 = [2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143] +; AVX512F-NEXT: vpternlogd $226, %zmm4, %zmm8, %zmm6 ; AVX512F-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm3 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm5, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm5, %zmm4 ; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsllw $6, %ymm0, %ymm3 ; AVX512F-NEXT: vpsrlw $2, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm7, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm7, %zmm4 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsllw $7, %ymm0, %ymm3 ; AVX512F-NEXT: vpsrlw $1, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm8, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm8, %zmm4 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 @@ -171,36 +171,36 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind { ; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2 ; AVX512VL-NEXT: vpsllw $4, %ymm2, %ymm3 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm4 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm5 = [1085102592571150095,1085102592571150095,1085102592571150095,1085102592571150095] -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm5, %ymm4 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm5 = [252645135,252645135,252645135,252645135,252645135,252645135,252645135,252645135] +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm5, %ymm4 ; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm3 ; AVX512VL-NEXT: vpsllw $5, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsllw $6, %ymm2, %ymm4 ; AVX512VL-NEXT: vpsrlw $2, %ymm2, %ymm6 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm7 = [4557430888798830399,4557430888798830399,4557430888798830399,4557430888798830399] -; AVX512VL-NEXT: vpternlogq $226, %ymm4, %ymm7, %ymm6 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm7 = [1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567,1061109567] +; AVX512VL-NEXT: vpternlogd $226, %ymm4, %ymm7, %ymm6 ; AVX512VL-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsllw $7, %ymm2, %ymm4 ; AVX512VL-NEXT: vpsrlw $1, %ymm2, %ymm6 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm8 = [9187201950435737471,9187201950435737471,9187201950435737471,9187201950435737471] -; AVX512VL-NEXT: vpternlogq $226, %ymm4, %ymm8, %ymm6 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm8 = [2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143,2139062143] +; AVX512VL-NEXT: vpternlogd $226, %ymm4, %ymm8, %ymm6 ; AVX512VL-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm5, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm5, %ymm4 ; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsllw $6, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsrlw $2, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm7, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm7, %ymm4 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsllw $7, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsrlw $1, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm8, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm8, %ymm4 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 @@ -721,7 +721,7 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind { ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: splatconstant_funnnel_v64i8: @@ -733,35 +733,35 @@ define <64 x i8> @splatconstant_funnnel_v64i8(<64 x i8> %x) nounwind { ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_funnnel_v64i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: retq ; ; AVX512VLBW-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_funnnel_v64i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLVBMI2-NEXT: retq %res = call <64 x i8> @llvm.fshr.v64i8(<64 x i8> %x, <64 x i8> %x, <64 x i8> ) ret <64 x i8> %res diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-4.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-4.ll index c9cbb0994810..e2195f1fc25a 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-4.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-4.ll @@ -156,64 +156,62 @@ define void @load_i8_stride4_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr define void @load_i8_stride4_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind { ; SSE-LABEL: load_i8_stride4_vf8: ; SSE: # %bb.0: -; SSE-NEXT: movdqa (%rdi), %xmm1 +; SSE-NEXT: movdqa (%rdi), %xmm3 ; SSE-NEXT: movdqa 16(%rdi), %xmm4 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,0,255,0,255,0,255,0] +; SSE-NEXT: movdqa %xmm4, %xmm1 ; SSE-NEXT: movdqa %xmm4, %xmm2 -; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: pand %xmm1, %xmm0 -; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm4[3,1,2,3,4,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm4 +; SSE-NEXT: pand %xmm3, %xmm0 +; SSE-NEXT: packuswb %xmm4, %xmm0 ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pxor %xmm7, %xmm7 -; SSE-NEXT: movdqa %xmm4, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm7[8],xmm2[9],xmm7[9],xmm2[10],xmm7[10],xmm2[11],xmm7[11],xmm2[12],xmm7[12],xmm2[13],xmm7[13],xmm2[14],xmm7[14],xmm2[15],xmm7[15] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm3[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm4, %xmm3 -; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3],xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm6[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1] -; SSE-NEXT: movdqa %xmm1, %xmm5 -; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm7[8],xmm5[9],xmm7[9],xmm5[10],xmm7[10],xmm5[11],xmm7[11],xmm5[12],xmm7[12],xmm5[13],xmm7[13],xmm5[14],xmm7[14],xmm5[15],xmm7[15] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm6[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm1, %xmm6 -; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7] -; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm6[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1] -; SSE-NEXT: packuswb %xmm8, %xmm7 -; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,3,2,3] -; SSE-NEXT: movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm8, %xmm4 -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm8, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm4, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] +; SSE-NEXT: pxor %xmm6, %xmm6 +; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm6[0],xmm2[1],xmm6[1],xmm2[2],xmm6[2],xmm2[3],xmm6[3],xmm2[4],xmm6[4],xmm2[5],xmm6[5],xmm2[6],xmm6[6],xmm2[7],xmm6[7] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm2[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1] +; SSE-NEXT: movdqa %xmm3, %xmm4 +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm6[8],xmm3[9],xmm6[9],xmm3[10],xmm6[10],xmm3[11],xmm6[11],xmm3[12],xmm6[12],xmm3[13],xmm6[13],xmm3[14],xmm6[14],xmm3[15],xmm6[15] +; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm3[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3],xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1] +; SSE-NEXT: packuswb %xmm7, %xmm6 +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,3,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,5,6,7] +; SSE-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm7, %xmm5 +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm7, %xmm8 +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm8[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm5, %xmm7 +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm7[0,3,2,3] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm5[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm6[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1] -; SSE-NEXT: packuswb %xmm3, %xmm4 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,3,2,3] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm4[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1] +; SSE-NEXT: packuswb %xmm2, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,3,2,3] ; SSE-NEXT: movq %xmm0, (%rsi) -; SSE-NEXT: movq %xmm7, (%rdx) -; SSE-NEXT: movq %xmm1, (%rcx) -; SSE-NEXT: movq %xmm2, (%r8) +; SSE-NEXT: movq %xmm6, (%rdx) +; SSE-NEXT: movq %xmm5, (%rcx) +; SSE-NEXT: movq %xmm1, (%r8) ; SSE-NEXT: retq ; ; AVX1-ONLY-LABEL: load_i8_stride4_vf8: @@ -296,111 +294,111 @@ define void @load_i8_stride4_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-LABEL: load_i8_stride4_vf16: ; SSE: # %bb.0: ; SSE-NEXT: movdqa (%rdi), %xmm1 -; SSE-NEXT: movdqa 16(%rdi), %xmm2 -; SSE-NEXT: movdqa 32(%rdi), %xmm8 -; SSE-NEXT: movdqa 48(%rdi), %xmm13 -; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,0,255,0,255,0,255,0] -; SSE-NEXT: movdqa %xmm13, %xmm3 -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: movdqa %xmm8, %xmm4 -; SSE-NEXT: pand %xmm0, %xmm4 -; SSE-NEXT: packuswb %xmm3, %xmm4 -; SSE-NEXT: movdqa %xmm2, %xmm3 -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: pand %xmm1, %xmm0 -; SSE-NEXT: packuswb %xmm3, %xmm0 -; SSE-NEXT: packuswb %xmm4, %xmm0 -; SSE-NEXT: pxor %xmm10, %xmm10 -; SSE-NEXT: movdqa %xmm13, %xmm3 -; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm10[8],xmm3[9],xmm10[9],xmm3[10],xmm10[10],xmm3[11],xmm10[11],xmm3[12],xmm10[12],xmm3[13],xmm10[13],xmm3[14],xmm10[14],xmm3[15],xmm10[15] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm13, %xmm4 -; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3],xmm4[4],xmm10[4],xmm4[5],xmm10[5],xmm4[6],xmm10[6],xmm4[7],xmm10[7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm6[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm5[0],xmm7[1],xmm5[1] -; SSE-NEXT: movdqa %xmm8, %xmm5 -; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm10[8],xmm5[9],xmm10[9],xmm5[10],xmm10[10],xmm5[11],xmm10[11],xmm5[12],xmm10[12],xmm5[13],xmm10[13],xmm5[14],xmm10[14],xmm5[15],xmm10[15] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm5[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm6[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm8, %xmm6 -; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm10[0],xmm6[1],xmm10[1],xmm6[2],xmm10[2],xmm6[3],xmm10[3],xmm6[4],xmm10[4],xmm6[5],xmm10[5],xmm6[6],xmm10[6],xmm6[7],xmm10[7] -; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm6[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm11[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm9[0],xmm14[1],xmm9[1] -; SSE-NEXT: packuswb %xmm7, %xmm14 -; SSE-NEXT: movdqa %xmm2, %xmm7 -; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm10[8],xmm7[9],xmm10[9],xmm7[10],xmm10[10],xmm7[11],xmm10[11],xmm7[12],xmm10[12],xmm7[13],xmm10[13],xmm7[14],xmm10[14],xmm7[15],xmm10[15] -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm7[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm9[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm2, %xmm9 -; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3],xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7] -; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm9[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm12[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm11[0],xmm15[1],xmm11[1] -; SSE-NEXT: movdqa %xmm1, %xmm12 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm1, %xmm11 -; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm10[8],xmm11[9],xmm10[9],xmm11[10],xmm10[10],xmm11[11],xmm10[11],xmm11[12],xmm10[12],xmm11[13],xmm10[13],xmm11[14],xmm10[14],xmm11[15],xmm10[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3],xmm12[4],xmm10[4],xmm12[5],xmm10[5],xmm12[6],xmm10[6],xmm12[7],xmm10[7] -; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm11[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm10[1,3,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm12[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm1[0],xmm10[1],xmm1[1] -; SSE-NEXT: packuswb %xmm15, %xmm10 -; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[0,3],xmm14[0,3] -; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm1, %xmm13 -; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm13[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm13[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm1, %xmm8 -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm13, %xmm8 +; SSE-NEXT: movdqa 16(%rdi), %xmm3 +; SSE-NEXT: movdqa 32(%rdi), %xmm9 +; SSE-NEXT: movdqa 48(%rdi), %xmm10 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0] +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: movdqa %xmm10, %xmm4 +; SSE-NEXT: movdqa %xmm10, %xmm7 +; SSE-NEXT: pand %xmm2, %xmm7 +; SSE-NEXT: movdqa %xmm9, %xmm5 +; SSE-NEXT: movdqa %xmm9, %xmm6 +; SSE-NEXT: movdqa %xmm9, %xmm11 +; SSE-NEXT: pand %xmm2, %xmm11 +; SSE-NEXT: packuswb %xmm7, %xmm11 +; SSE-NEXT: movdqa %xmm3, %xmm7 +; SSE-NEXT: movdqa %xmm3, %xmm8 +; SSE-NEXT: movdqa %xmm3, %xmm12 +; SSE-NEXT: pand %xmm2, %xmm12 ; SSE-NEXT: pand %xmm1, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload -; SSE-NEXT: pand %xmm1, %xmm13 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,7,6,5,4] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm13[3,1,2,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm12, %xmm2 +; SSE-NEXT: packuswb %xmm11, %xmm2 +; SSE-NEXT: pxor %xmm11, %xmm11 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm11[8],xmm0[9],xmm11[9],xmm0[10],xmm11[10],xmm0[11],xmm11[11],xmm0[12],xmm11[12],xmm0[13],xmm11[13],xmm0[14],xmm11[14],xmm0[15],xmm11[15] +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm0[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm11[0],xmm4[1],xmm11[1],xmm4[2],xmm11[2],xmm4[3],xmm11[3],xmm4[4],xmm11[4],xmm4[5],xmm11[5],xmm4[6],xmm11[6],xmm4[7],xmm11[7] +; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm4[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm13[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm12[0],xmm13[1],xmm12[1] +; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm11[8],xmm5[9],xmm11[9],xmm5[10],xmm11[10],xmm5[11],xmm11[11],xmm5[12],xmm11[12],xmm5[13],xmm11[13],xmm5[14],xmm11[14],xmm5[15],xmm11[15] +; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm5[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm11[0],xmm6[1],xmm11[1],xmm6[2],xmm11[2],xmm6[3],xmm11[3],xmm6[4],xmm11[4],xmm6[5],xmm11[5],xmm6[6],xmm11[6],xmm6[7],xmm11[7] +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm6[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm14[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm12[0],xmm14[1],xmm12[1] +; SSE-NEXT: packuswb %xmm13, %xmm14 +; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm11[8],xmm7[9],xmm11[9],xmm7[10],xmm11[10],xmm7[11],xmm11[11],xmm7[12],xmm11[12],xmm7[13],xmm11[13],xmm7[14],xmm11[14],xmm7[15],xmm11[15] +; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm7[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm11[0],xmm8[1],xmm11[1],xmm8[2],xmm11[2],xmm8[3],xmm11[3],xmm8[4],xmm11[4],xmm8[5],xmm11[5],xmm8[6],xmm11[6],xmm8[7],xmm11[7] +; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm8[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm13[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm12[0],xmm15[1],xmm12[1] +; SSE-NEXT: movdqa %xmm1, %xmm12 +; SSE-NEXT: movdqa %xmm1, %xmm13 +; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm11[8],xmm13[9],xmm11[9],xmm13[10],xmm11[10],xmm13[11],xmm11[11],xmm13[12],xmm11[12],xmm13[13],xmm11[13],xmm13[14],xmm11[14],xmm13[15],xmm11[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm11[0],xmm12[1],xmm11[1],xmm12[2],xmm11[2],xmm12[3],xmm11[3],xmm12[4],xmm11[4],xmm12[5],xmm11[5],xmm12[6],xmm11[6],xmm12[7],xmm11[7] +; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm13[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[1,3,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm12[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm0[0],xmm11[1],xmm0[1] +; SSE-NEXT: packuswb %xmm15, %xmm11 +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[3,1,2,3,4,5,6,7] +; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[0,3],xmm14[0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,7] +; SSE-NEXT: movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm10, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm10, %xmm9 +; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm0, %xmm9 +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm10, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm1 -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm8[0,3] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] +; SSE-NEXT: packuswb %xmm0, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm9[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE-NEXT: # xmm0 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm4[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm5[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm5[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm6[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] ; SSE-NEXT: packuswb %xmm3, %xmm4 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm7[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm9[3,1,2,3] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm7[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm8[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm11[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm13[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm12[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1] +; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1] ; SSE-NEXT: packuswb %xmm3, %xmm5 ; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,3],xmm4[0,3] -; SSE-NEXT: movdqa %xmm0, (%rsi) -; SSE-NEXT: movaps %xmm10, (%rdx) +; SSE-NEXT: movdqa %xmm2, (%rsi) +; SSE-NEXT: movaps %xmm11, (%rdx) ; SSE-NEXT: movaps %xmm1, (%rcx) ; SSE-NEXT: movaps %xmm5, (%r8) ; SSE-NEXT: retq @@ -530,249 +528,247 @@ define void @load_i8_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE: # %bb.0: ; SSE-NEXT: subq $136, %rsp ; SSE-NEXT: movdqa 64(%rdi), %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 80(%rdi), %xmm5 -; SSE-NEXT: movdqa 96(%rdi), %xmm15 -; SSE-NEXT: movdqa 112(%rdi), %xmm14 -; SSE-NEXT: movdqa (%rdi), %xmm12 -; SSE-NEXT: movdqa 16(%rdi), %xmm11 -; SSE-NEXT: movdqa 32(%rdi), %xmm7 -; SSE-NEXT: movdqa 48(%rdi), %xmm3 -; SSE-NEXT: movdqa {{.*#+}} xmm6 = [255,0,255,0,255,0,255,0] -; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: pand %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: pand %xmm6, %xmm1 -; SSE-NEXT: packuswb %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm11, %xmm0 -; SSE-NEXT: pand %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm12, %xmm2 -; SSE-NEXT: pand %xmm6, %xmm2 -; SSE-NEXT: packuswb %xmm0, %xmm2 -; SSE-NEXT: packuswb %xmm1, %xmm2 +; SSE-NEXT: movdqa 80(%rdi), %xmm9 +; SSE-NEXT: movdqa 96(%rdi), %xmm11 +; SSE-NEXT: movdqa 112(%rdi), %xmm12 +; SSE-NEXT: movdqa (%rdi), %xmm13 +; SSE-NEXT: movdqa 16(%rdi), %xmm14 +; SSE-NEXT: movdqa 32(%rdi), %xmm2 +; SSE-NEXT: movdqa 48(%rdi), %xmm10 +; SSE-NEXT: movdqa {{.*#+}} xmm8 = [255,0,255,0,255,0,255,0] +; SSE-NEXT: movdqa %xmm10, %xmm3 +; SSE-NEXT: movdqa %xmm10, %xmm5 +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: pand %xmm8, %xmm0 +; SSE-NEXT: movdqa %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm2, %xmm1 +; SSE-NEXT: movdqa %xmm2, %xmm15 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm8, %xmm1 +; SSE-NEXT: packuswb %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: pand %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm15, %xmm1 -; SSE-NEXT: pand %xmm6, %xmm1 +; SSE-NEXT: pand %xmm8, %xmm0 +; SSE-NEXT: movdqa %xmm13, %xmm7 +; SSE-NEXT: pand %xmm8, %xmm7 +; SSE-NEXT: packuswb %xmm0, %xmm7 +; SSE-NEXT: packuswb %xmm1, %xmm7 +; SSE-NEXT: movdqa %xmm12, %xmm0 +; SSE-NEXT: pand %xmm8, %xmm0 +; SSE-NEXT: movdqa %xmm11, %xmm1 +; SSE-NEXT: pand %xmm8, %xmm1 ; SSE-NEXT: packuswb %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm5, %xmm2 -; SSE-NEXT: pand %xmm6, %xmm0 -; SSE-NEXT: pand %xmm4, %xmm6 -; SSE-NEXT: packuswb %xmm0, %xmm6 -; SSE-NEXT: packuswb %xmm1, %xmm6 -; SSE-NEXT: pxor %xmm4, %xmm4 -; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SSE-NEXT: movdqa %xmm9, %xmm0 +; SSE-NEXT: pand %xmm8, %xmm0 +; SSE-NEXT: pand %xmm4, %xmm8 +; SSE-NEXT: packuswb %xmm0, %xmm8 +; SSE-NEXT: packuswb %xmm1, %xmm8 +; SSE-NEXT: pxor %xmm2, %xmm2 +; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15] +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm3, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7] +; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15] +; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm15, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,1,1,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7] ; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1] -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1] -; SSE-NEXT: packuswb %xmm5, %xmm8 -; SSE-NEXT: movdqa %xmm11, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] +; SSE-NEXT: packuswb %xmm3, %xmm5 +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm14, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm11, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSE-NEXT: movdqa %xmm14, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm5[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1] -; SSE-NEXT: movdqa %xmm12, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm13, %xmm14 +; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] ; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm12, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm5[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm0[0],xmm13[1],xmm0[1] -; SSE-NEXT: packuswb %xmm9, %xmm13 -; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,3],xmm8[0,3] -; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm2[0],xmm14[1],xmm2[1],xmm14[2],xmm2[2],xmm14[3],xmm2[3],xmm14[4],xmm2[4],xmm14[5],xmm2[5],xmm14[6],xmm2[6],xmm14[7],xmm2[7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm14[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm6[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] +; SSE-NEXT: packuswb %xmm3, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm5[0,3] +; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm12, %xmm15 +; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm12, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm14, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1] -; SSE-NEXT: movdqa %xmm15, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm15, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1] -; SSE-NEXT: packuswb %xmm8, %xmm0 -; SSE-NEXT: movdqa %xmm2, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm2, %xmm10 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm9[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Reload -; SSE-NEXT: movdqa %xmm5, %xmm9 -; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15] -; SSE-NEXT: movdqa %xmm5, %xmm8 -; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3],xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm9[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm4[1,3,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm8[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm2[0],xmm15[1],xmm2[1],xmm15[2],xmm2[2],xmm15[3],xmm2[3],xmm15[4],xmm2[4],xmm15[5],xmm2[5],xmm15[6],xmm2[6],xmm15[7],xmm2[7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm15[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm11, %xmm12 +; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm11, %xmm13 +; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm2[8],xmm13[9],xmm2[9],xmm13[10],xmm2[10],xmm13[11],xmm2[11],xmm13[12],xmm2[12],xmm13[13],xmm2[13],xmm13[14],xmm2[14],xmm13[15],xmm2[15] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm13[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm0[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm2[0],xmm12[1],xmm2[1],xmm12[2],xmm2[2],xmm12[3],xmm2[3],xmm12[4],xmm2[4],xmm12[5],xmm2[5],xmm12[6],xmm2[6],xmm12[7],xmm2[7] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm12[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm5[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1] +; SSE-NEXT: packuswb %xmm3, %xmm0 +; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm9, %xmm11 +; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm2[8],xmm11[9],xmm2[9],xmm11[10],xmm2[10],xmm11[11],xmm2[11],xmm11[12],xmm2[12],xmm11[13],xmm2[13],xmm11[14],xmm2[14],xmm11[15],xmm2[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3],xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm9[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm5[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] +; SSE-NEXT: movdqa %xmm4, %xmm5 +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm4, %xmm6 +; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm6[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,3,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm3[1,3,2,3,4,5,6,7] ; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1] ; SSE-NEXT: packuswb %xmm1, %xmm4 ; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,3],xmm0[0,3] -; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,7,5,6,7] +; SSE-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm3, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm2, %xmm1 +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload +; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload +; SSE-NEXT: # xmm10 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm10 +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm2, %xmm10 +; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[0,3],xmm1[0,3] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload +; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm1 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm0, %xmm7 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm7[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm2 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,0,3,2,4,5,6,7] ; SSE-NEXT: packuswb %xmm1, %xmm2 -; SSE-NEXT: pand %xmm0, %xmm11 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm11[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm1[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm0, %xmm12 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm3, %xmm1 -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm2[0,3] -; SSE-NEXT: pand %xmm0, %xmm14 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm14[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm0, %xmm15 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm15[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm3[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm7 -; SSE-NEXT: pand %xmm0, %xmm10 -; SSE-NEXT: pand %xmm0, %xmm5 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[3,1,2,3,4,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm1 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm5[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm2[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm0, %xmm3 -; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm7[0,3] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload -; SSE-NEXT: # xmm2 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload -; SSE-NEXT: # xmm7 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1] -; SSE-NEXT: packuswb %xmm2, %xmm7 -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7] +; SSE-NEXT: pand %xmm3, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm1, %xmm0 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,3],xmm2[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload ; SSE-NEXT: # xmm2 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1] -; SSE-NEXT: pshufd $231, (%rsp), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm11[0],xmm0[1],xmm11[1] -; SSE-NEXT: packuswb %xmm2, %xmm0 -; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,3],xmm7[0,3] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1] +; SSE-NEXT: packuswb %xmm2, %xmm3 +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload ; SSE-NEXT: # xmm2 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload -; SSE-NEXT: # xmm7 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload -; SSE-NEXT: # xmm2 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload -; SSE-NEXT: # xmm11 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1] -; SSE-NEXT: packuswb %xmm7, %xmm11 -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload -; SSE-NEXT: # xmm2 = mem[3,1,2,3] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: pshufd $231, (%rsp), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm14[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm14[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm14 = xmm14[0],xmm1[0],xmm14[1],xmm1[1] +; SSE-NEXT: packuswb %xmm2, %xmm14 +; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[0,3],xmm3[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload +; SSE-NEXT: # xmm1 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm15[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload -; SSE-NEXT: # xmm7 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm13[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm12[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1] +; SSE-NEXT: packuswb %xmm2, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm11[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm9[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm2[0],xmm8[1],xmm2[1] -; SSE-NEXT: packuswb %xmm7, %xmm8 -; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,3],xmm11[0,3] -; SSE-NEXT: movdqa %xmm6, 16(%rsi) -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload -; SSE-NEXT: movaps %xmm2, (%rsi) +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,1,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1] +; SSE-NEXT: packuswb %xmm2, %xmm5 +; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,3],xmm3[0,3] +; SSE-NEXT: movdqa %xmm8, 16(%rsi) +; SSE-NEXT: movdqa %xmm7, (%rsi) ; SSE-NEXT: movaps %xmm4, 16(%rdx) -; SSE-NEXT: movaps %xmm13, (%rdx) -; SSE-NEXT: movaps %xmm3, 16(%rcx) -; SSE-NEXT: movaps %xmm1, (%rcx) -; SSE-NEXT: movaps %xmm8, 16(%r8) -; SSE-NEXT: movaps %xmm0, (%r8) +; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: movaps %xmm1, (%rdx) +; SSE-NEXT: movaps %xmm0, 16(%rcx) +; SSE-NEXT: movaps %xmm10, (%rcx) +; SSE-NEXT: movaps %xmm5, 16(%r8) +; SSE-NEXT: movaps %xmm14, (%r8) ; SSE-NEXT: addq $136, %rsp ; SSE-NEXT: retq ; @@ -1023,522 +1019,518 @@ define void @load_i8_stride4_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr define void @load_i8_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3) nounwind { ; SSE-LABEL: load_i8_stride4_vf64: ; SSE: # %bb.0: -; SSE-NEXT: subq $664, %rsp # imm = 0x298 +; SSE-NEXT: subq $600, %rsp # imm = 0x258 ; SSE-NEXT: movdqa 16(%rdi), %xmm8 ; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 32(%rdi), %xmm6 +; SSE-NEXT: movdqa 32(%rdi), %xmm15 ; SSE-NEXT: movdqa 48(%rdi), %xmm14 -; SSE-NEXT: movdqa 128(%rdi), %xmm15 -; SSE-NEXT: movdqa 144(%rdi), %xmm10 -; SSE-NEXT: movdqa 160(%rdi), %xmm11 -; SSE-NEXT: movdqa 176(%rdi), %xmm3 -; SSE-NEXT: movdqa 64(%rdi), %xmm13 -; SSE-NEXT: movdqa 80(%rdi), %xmm7 +; SSE-NEXT: movdqa 128(%rdi), %xmm4 +; SSE-NEXT: movdqa 144(%rdi), %xmm7 +; SSE-NEXT: movdqa 160(%rdi), %xmm10 +; SSE-NEXT: movdqa 176(%rdi), %xmm12 +; SSE-NEXT: movdqa 64(%rdi), %xmm6 +; SSE-NEXT: movdqa 80(%rdi), %xmm13 ; SSE-NEXT: movdqa 96(%rdi), %xmm2 ; SSE-NEXT: movdqa 112(%rdi), %xmm1 -; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,0,255,0,255,0,255,0] +; SSE-NEXT: movdqa {{.*#+}} xmm9 = [255,0,255,0,255,0,255,0] ; SSE-NEXT: movdqa %xmm1, %xmm0 -; SSE-NEXT: movdqa %xmm1, %xmm4 -; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: movdqa %xmm1, %xmm3 +; SSE-NEXT: pand %xmm9, %xmm0 ; SSE-NEXT: movdqa %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm2, %xmm9 -; SSE-NEXT: pand %xmm5, %xmm1 +; SSE-NEXT: movdqa %xmm2, %xmm5 +; SSE-NEXT: pand %xmm9, %xmm1 ; SSE-NEXT: packuswb %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm13, %xmm2 -; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: pand %xmm9, %xmm0 +; SSE-NEXT: movdqa %xmm6, %xmm2 +; SSE-NEXT: pand %xmm9, %xmm2 ; SSE-NEXT: packuswb %xmm0, %xmm2 ; SSE-NEXT: packuswb %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm11, %xmm1 -; SSE-NEXT: pand %xmm5, %xmm1 +; SSE-NEXT: movdqa %xmm12, %xmm0 +; SSE-NEXT: pand %xmm9, %xmm0 +; SSE-NEXT: movdqa %xmm10, %xmm1 +; SSE-NEXT: pand %xmm9, %xmm1 ; SSE-NEXT: packuswb %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm15, %xmm2 -; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: pand %xmm9, %xmm0 +; SSE-NEXT: movdqa %xmm4, %xmm2 +; SSE-NEXT: pand %xmm9, %xmm2 ; SSE-NEXT: packuswb %xmm0, %xmm2 ; SSE-NEXT: packuswb %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm6, %xmm1 -; SSE-NEXT: pand %xmm5, %xmm1 +; SSE-NEXT: pand %xmm9, %xmm0 +; SSE-NEXT: movdqa %xmm15, %xmm1 +; SSE-NEXT: pand %xmm9, %xmm1 ; SSE-NEXT: packuswb %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 -; SSE-NEXT: movdqa (%rdi), %xmm12 -; SSE-NEXT: movdqa %xmm12, %xmm2 -; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: pand %xmm9, %xmm0 +; SSE-NEXT: movdqa (%rdi), %xmm11 +; SSE-NEXT: movdqa %xmm11, %xmm2 +; SSE-NEXT: pand %xmm9, %xmm2 ; SSE-NEXT: packuswb %xmm0, %xmm2 ; SSE-NEXT: packuswb %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 240(%rdi), %xmm8 -; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: movdqa 240(%rdi), %xmm0 +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm9, %xmm0 ; SSE-NEXT: movdqa 224(%rdi), %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm5, %xmm1 +; SSE-NEXT: pand %xmm9, %xmm1 ; SSE-NEXT: packuswb %xmm0, %xmm1 ; SSE-NEXT: movdqa 208(%rdi), %xmm0 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: pand %xmm9, %xmm0 ; SSE-NEXT: movdqa 192(%rdi), %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm2, %xmm5 -; SSE-NEXT: packuswb %xmm0, %xmm5 -; SSE-NEXT: packuswb %xmm1, %xmm5 -; SSE-NEXT: pxor %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm4, %xmm1 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: pand %xmm2, %xmm9 +; SSE-NEXT: packuswb %xmm0, %xmm9 +; SSE-NEXT: packuswb %xmm1, %xmm9 +; SSE-NEXT: movdqa %xmm3, %xmm1 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm3, %xmm0 +; SSE-NEXT: pxor %xmm8, %xmm8 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: movdqa %xmm9, %xmm1 -; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm9, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm5, %xmm1 +; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm5, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1] -; SSE-NEXT: packuswb %xmm4, %xmm9 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1] +; SSE-NEXT: packuswb %xmm3, %xmm5 +; SSE-NEXT: movdqa %xmm13, %xmm1 +; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: movdqa %xmm13, %xmm1 -; SSE-NEXT: movdqa %xmm13, (%rsp) # 16-byte Spill -; SSE-NEXT: movdqa %xmm13, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm6, %xmm1 +; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm6, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm13[1,3,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm6[1,3,2,3,4,5,6,7] ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE-NEXT: packuswb %xmm4, %xmm1 -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm9[0,3] +; SSE-NEXT: packuswb %xmm3, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm5[0,3] ; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: movdqa %xmm12, %xmm1 +; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm12, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm11, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm10, %xmm1 +; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm2[0],xmm11[1],xmm2[1],xmm11[2],xmm2[2],xmm11[3],xmm2[3],xmm11[4],xmm2[4],xmm11[5],xmm2[5],xmm11[6],xmm2[6],xmm11[7],xmm2[7] -; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm11[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm0[0],xmm9[1],xmm0[1] -; SSE-NEXT: packuswb %xmm4, %xmm9 -; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm5[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1] +; SSE-NEXT: packuswb %xmm3, %xmm6 +; SSE-NEXT: movdqa %xmm7, (%rsp) # 16-byte Spill +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3],xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7] -; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm10[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: movdqa %xmm15, %xmm1 -; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm15, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: movdqa %xmm7, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm4, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] +; SSE-NEXT: movdqa %xmm4, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1],xmm1[2],xmm8[2],xmm1[3],xmm8[3],xmm1[4],xmm8[4],xmm1[5],xmm8[5],xmm1[6],xmm8[6],xmm1[7],xmm8[7] ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm13[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE-NEXT: packuswb %xmm4, %xmm1 -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm9[0,3] -; SSE-NEXT: movaps %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm5[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm0[0],xmm10[1],xmm0[1] +; SSE-NEXT: packuswb %xmm3, %xmm10 +; SSE-NEXT: shufps {{.*#+}} xmm10 = xmm10[0,3],xmm6[0,3] +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm14, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: movdqa %xmm6, %xmm7 -; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm8[0],xmm14[1],xmm8[1],xmm14[2],xmm8[2],xmm14[3],xmm8[3],xmm14[4],xmm8[4],xmm14[5],xmm8[5],xmm14[6],xmm8[6],xmm14[7],xmm8[7] +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm14[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] +; SSE-NEXT: movdqa %xmm15, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm0[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm9[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm13[0],xmm0[1],xmm13[1] -; SSE-NEXT: packuswb %xmm4, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload -; SSE-NEXT: movdqa %xmm6, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm6, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm9[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1] -; SSE-NEXT: movdqa %xmm12, %xmm3 -; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm2[8],xmm3[9],xmm2[9],xmm3[10],xmm2[10],xmm3[11],xmm2[11],xmm3[12],xmm2[12],xmm3[13],xmm2[13],xmm3[14],xmm2[14],xmm3[15],xmm2[15] -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm4[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm12, %xmm4 -; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7] -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm4[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm15 = xmm15[0],xmm3[0],xmm15[1],xmm3[1] -; SSE-NEXT: packuswb %xmm1, %xmm15 -; SSE-NEXT: shufps {{.*#+}} xmm15 = xmm15[0,3],xmm0[0,3] -; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm0[1,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm15, %xmm12 +; SSE-NEXT: movdqa %xmm15, %xmm14 +; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm8[0],xmm12[1],xmm8[1],xmm12[2],xmm8[2],xmm12[3],xmm8[3],xmm12[4],xmm8[4],xmm12[5],xmm8[5],xmm12[6],xmm8[6],xmm12[7],xmm8[7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm12[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm6[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1] +; SSE-NEXT: packuswb %xmm3, %xmm0 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload +; SSE-NEXT: movdqa %xmm15, %xmm13 +; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm8[8],xmm13[9],xmm8[9],xmm13[10],xmm8[10],xmm13[11],xmm8[11],xmm13[12],xmm8[12],xmm13[13],xmm8[13],xmm13[14],xmm8[14],xmm13[15],xmm8[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm13[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm8[0],xmm15[1],xmm8[1],xmm15[2],xmm8[2],xmm15[3],xmm8[3],xmm15[4],xmm8[4],xmm15[5],xmm8[5],xmm15[6],xmm8[6],xmm15[7],xmm8[7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm15[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm6[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] +; SSE-NEXT: movdqa %xmm11, %xmm7 +; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm8[8],xmm7[9],xmm8[9],xmm7[10],xmm8[10],xmm7[11],xmm8[11],xmm7[12],xmm8[12],xmm7[13],xmm8[13],xmm7[14],xmm8[14],xmm7[15],xmm8[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm7[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm11, %xmm6 +; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1],xmm6[2],xmm8[2],xmm6[3],xmm8[3],xmm6[4],xmm8[4],xmm6[5],xmm8[5],xmm6[6],xmm8[6],xmm6[7],xmm8[7] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1] +; SSE-NEXT: packuswb %xmm1, %xmm5 +; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,3],xmm0[0,3] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm8, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] +; SSE-NEXT: movdqa %xmm1, %xmm3 +; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3],xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7] +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7] ; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,3,2,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm10, %xmm3 -; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7] -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] +; SSE-NEXT: movdqa %xmm2, %xmm4 +; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm8[0],xmm4[1],xmm8[1],xmm4[2],xmm8[2],xmm4[3],xmm8[3],xmm4[4],xmm8[4],xmm4[5],xmm8[5],xmm4[6],xmm8[6],xmm4[7],xmm8[7] +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,3,2,3,4,5,6,7] ; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1] ; SSE-NEXT: packuswb %xmm1, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload -; SSE-NEXT: movdqa %xmm11, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm8[8],xmm0[9],xmm8[9],xmm0[10],xmm8[10],xmm0[11],xmm8[11],xmm0[12],xmm8[12],xmm0[13],xmm8[13],xmm0[14],xmm8[14],xmm0[15],xmm8[15] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm0[0,1,1,3,4,5,6,7] -; SSE-NEXT: movdqa %xmm11, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7] ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm1[0,1,1,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; SSE-NEXT: movdqa %xmm9, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm9, %xmm4 -; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[0,1,1,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE-NEXT: movdqa %xmm0, %xmm4 +; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm8[8],xmm4[9],xmm8[9],xmm4[10],xmm8[10],xmm4[11],xmm8[11],xmm4[12],xmm8[12],xmm4[13],xmm8[13],xmm4[14],xmm8[14],xmm4[15],xmm8[15] ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[1,3,2,3,4,5,6,7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1],xmm0[2],xmm8[2],xmm0[3],xmm8[3],xmm0[4],xmm8[4],xmm0[5],xmm8[5],xmm0[6],xmm8[6],xmm0[7],xmm8[7] +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm4[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm1[1,3,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,3,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1] -; SSE-NEXT: packuswb %xmm0, %xmm1 +; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1] +; SSE-NEXT: packuswb %xmm2, %xmm1 ; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,3],xmm3[0,3] -; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm0, %xmm14 +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE-NEXT: # xmm0 = mem[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm14[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,7,5,6,7] +; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm0, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm0, %xmm7 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm7[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm3 -; SSE-NEXT: pand %xmm0, %xmm6 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm6[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm0, %xmm12 -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm12[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm4[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm12 -; SSE-NEXT: shufps {{.*#+}} xmm12 = xmm12[0,3],xmm3[0,3] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload ; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm4 +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload +; SSE-NEXT: # xmm2 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm11[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,6,5,4] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload ; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm2[0,1,2,3,7,6,5,4] -; SSE-NEXT: movdqa (%rsp), %xmm2 # 16-byte Reload +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm2[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm3[0,1,2,3,7,5,6,7] ; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm4, %xmm2 -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,3],xmm3[0,3] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm8, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,3],xmm4[0,3] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload +; SSE-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm3 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: pand %xmm0, %xmm4 -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm4 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,0,3,2,4,5,6,7] ; SSE-NEXT: packuswb %xmm3, %xmm4 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload +; SSE-NEXT: # xmm8 = mem[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm3 ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm3[0,1,2,3,7,6,5,4] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm8 +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm8[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm11 +; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[0,3],xmm4[0,3] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload +; SSE-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] ; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm4 +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm4 +; SSE-NEXT: pshuflw $231, (%rsp), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload +; SSE-NEXT: # xmm8 = mem[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm6, %xmm3 -; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,3],xmm4[0,3] +; SSE-NEXT: pand %xmm0, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,5,6,7] ; SSE-NEXT: pand %xmm0, %xmm8 -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm8[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,6,5,4] -; SSE-NEXT: pand %xmm0, %xmm10 -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm10[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm6[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm4, %xmm7 -; SSE-NEXT: pand %xmm0, %xmm11 -; SSE-NEXT: pand %xmm0, %xmm9 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,6,5,4] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm9[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm8[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm14 +; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[0,3],xmm4[0,3] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload +; SSE-NEXT: # xmm4 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,6,5,4] ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm4 ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm4[1,0,3,2,4,5,6,7] -; SSE-NEXT: packuswb %xmm0, %xmm6 -; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,3],xmm7[0,3] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm4 +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload +; SSE-NEXT: # xmm8 = mem[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm3 +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm8 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm0[0,1,2,3,7,6,5,4] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm8[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,0,3,2,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm0 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,3],xmm4[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload -; SSE-NEXT: # xmm7 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1] -; SSE-NEXT: packuswb %xmm4, %xmm7 -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm12[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm8 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm13[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm15[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: # xmm0 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1] -; SSE-NEXT: packuswb %xmm4, %xmm0 -; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,3],xmm7[0,3] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm7[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm6 +; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,3],xmm8[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload ; SSE-NEXT: # xmm7 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload -; SSE-NEXT: # xmm8 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1] -; SSE-NEXT: packuswb %xmm7, %xmm8 +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm7[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm8 +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload -; SSE-NEXT: # xmm7 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm7[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload ; SSE-NEXT: # xmm7 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1] -; SSE-NEXT: packuswb %xmm10, %xmm7 +; SSE-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm7 ; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,3],xmm8[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload ; SSE-NEXT: # xmm8 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; SSE-NEXT: # xmm10 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1] -; SSE-NEXT: packuswb %xmm8, %xmm10 +; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm8[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm3[0],xmm12[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm12 +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload -; SSE-NEXT: # xmm8 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm8[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload ; SSE-NEXT: # xmm8 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1] -; SSE-NEXT: packuswb %xmm11, %xmm8 -; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,3],xmm10[0,3] +; SSE-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm3[0],xmm8[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm8 +; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[0,3],xmm12[0,3] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; SSE-NEXT: # xmm10 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload -; SSE-NEXT: # xmm11 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1] -; SSE-NEXT: packuswb %xmm10, %xmm11 +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload +; SSE-NEXT: # xmm12 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm3[0],xmm12[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm12 +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload ; SSE-NEXT: # xmm4 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,3,1,4,5,6,7] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; SSE-NEXT: # xmm10 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,1,3,1,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1] -; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: # xmm4 = mem[3,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1] +; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: # xmm3 = mem[3,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload ; SSE-NEXT: # xmm13 = mem[3,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm13[3,1,2,3,4,5,6,7] -; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm4[0],xmm13[1],xmm4[1] -; SSE-NEXT: packuswb %xmm10, %xmm13 -; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,3],xmm11[0,3] -; SSE-NEXT: movdqa %xmm5, 48(%rsi) -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: movaps %xmm4, (%rsi) -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: movaps %xmm4, 32(%rsi) -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: movaps %xmm4, 16(%rsi) +; SSE-NEXT: punpckldq {{.*#+}} xmm13 = xmm13[0],xmm3[0],xmm13[1],xmm3[1] +; SSE-NEXT: packuswb %xmm4, %xmm13 +; SSE-NEXT: shufps {{.*#+}} xmm13 = xmm13[0,3],xmm12[0,3] +; SSE-NEXT: movdqa %xmm9, 48(%rsi) +; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: movaps %xmm3, (%rsi) +; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: movaps %xmm3, 32(%rsi) +; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: movaps %xmm3, 16(%rsi) ; SSE-NEXT: movaps %xmm1, 48(%rdx) -; SSE-NEXT: movaps %xmm15, (%rdx) -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: movaps %xmm1, 32(%rdx) +; SSE-NEXT: movaps %xmm5, (%rdx) +; SSE-NEXT: movaps %xmm10, 32(%rdx) ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; SSE-NEXT: movaps %xmm1, 16(%rdx) -; SSE-NEXT: movaps %xmm6, 48(%rcx) -; SSE-NEXT: movaps %xmm3, 32(%rcx) -; SSE-NEXT: movaps %xmm2, 16(%rcx) -; SSE-NEXT: movaps %xmm12, (%rcx) +; SSE-NEXT: movaps %xmm0, 48(%rcx) +; SSE-NEXT: movaps %xmm14, 32(%rcx) +; SSE-NEXT: movaps %xmm11, 16(%rcx) +; SSE-NEXT: movaps %xmm2, (%rcx) ; SSE-NEXT: movaps %xmm13, 48(%r8) ; SSE-NEXT: movaps %xmm8, 32(%r8) ; SSE-NEXT: movaps %xmm7, 16(%r8) -; SSE-NEXT: movaps %xmm0, (%r8) -; SSE-NEXT: addq $664, %rsp # imm = 0x298 +; SSE-NEXT: movaps %xmm6, (%r8) +; SSE-NEXT: addq $600, %rsp # imm = 0x258 ; SSE-NEXT: retq ; ; AVX1-ONLY-LABEL: load_i8_stride4_vf64: diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll index a2ad944c66f4..be34bbd25402 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i8-stride-6.ll @@ -104,9 +104,9 @@ define void @load_i8_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pand %xmm0, %xmm2 ; SSE-NEXT: pandn %xmm1, %xmm0 ; SSE-NEXT: por %xmm2, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm2, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [16711935,16711935,16711935,16711935] +; SSE-NEXT: pand %xmm2, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,1,4,5,6,7] @@ -130,10 +130,9 @@ define void @load_i8_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm9[2],xmm7[3],xmm9[3] ; SSE-NEXT: packuswb %xmm7, %xmm7 ; SSE-NEXT: por %xmm7, %xmm4 -; SSE-NEXT: movaps %xmm6, %xmm7 -; SSE-NEXT: andps %xmm2, %xmm7 -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm6[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,7,6,7] +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm7 ; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[1,2,3,0,4,5,6,7] ; SSE-NEXT: packuswb %xmm7, %xmm7 @@ -145,8 +144,8 @@ define void @load_i8_stride6_vf4(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3] ; SSE-NEXT: packuswb %xmm6, %xmm6 ; SSE-NEXT: por %xmm1, %xmm8 -; SSE-NEXT: pand %xmm8, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,1,2,0] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[3,1,2,0] +; SSE-NEXT: pand %xmm2, %xmm1 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,0,3,4,5,6,7] @@ -225,10 +224,9 @@ define void @load_i8_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pand %xmm8, %xmm1 ; SSE-NEXT: pandn %xmm3, %xmm8 ; SSE-NEXT: por %xmm1, %xmm8 -; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: movdqa %xmm8, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[0,2,1,3] +; SSE-NEXT: movdqa {{.*#+}} xmm5 = [16711935,16711935,16711935,16711935] ; SSE-NEXT: pand %xmm5, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7] @@ -274,10 +272,9 @@ define void @load_i8_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: movdqa %xmm4, %xmm12 ; SSE-NEXT: pand %xmm11, %xmm12 ; SSE-NEXT: por %xmm9, %xmm12 -; SSE-NEXT: movdqa %xmm12, %xmm9 -; SSE-NEXT: pand %xmm5, %xmm9 -; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm12[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm5, %xmm9 ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[1,2,3,0,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm13 = xmm9[0,1,2,3,5,5,5,5] @@ -318,8 +315,8 @@ define void @load_i8_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pand %xmm11, %xmm3 ; SSE-NEXT: pandn %xmm4, %xmm11 ; SSE-NEXT: por %xmm3, %xmm11 -; SSE-NEXT: pand %xmm11, %xmm5 -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm5[3,1,2,0] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[3,1,2,0] +; SSE-NEXT: pand %xmm5, %xmm3 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,1,0,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm3[2,1,0,3,4,5,6,7] @@ -470,79 +467,81 @@ define void @load_i8_stride6_vf8(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5) nounwind { ; SSE-LABEL: load_i8_stride6_vf16: ; SSE: # %bb.0: -; SSE-NEXT: movdqa 64(%rdi), %xmm11 -; SSE-NEXT: movdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 64(%rdi), %xmm10 +; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa (%rdi), %xmm5 -; SSE-NEXT: movdqa 16(%rdi), %xmm2 -; SSE-NEXT: movdqa 32(%rdi), %xmm8 +; SSE-NEXT: movdqa 16(%rdi), %xmm1 +; SSE-NEXT: movdqa 32(%rdi), %xmm7 ; SSE-NEXT: movdqa 48(%rdi), %xmm6 ; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,0,65535,65535,0,65535,65535] -; SSE-NEXT: movdqa %xmm4, %xmm7 -; SSE-NEXT: pandn %xmm8, %xmm7 -; SSE-NEXT: movdqa {{.*#+}} xmm10 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: pandn %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: pandn %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pandn %xmm7, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,0,65535,65535,0,65535,65535,0] +; SSE-NEXT: movdqa %xmm2, %xmm3 +; SSE-NEXT: pandn %xmm6, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm4, %xmm3 +; SSE-NEXT: pandn %xmm6, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm4, %xmm6 -; SSE-NEXT: por %xmm7, %xmm6 -; SSE-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: por %xmm0, %xmm6 ; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm0 +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,2,3] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] ; SSE-NEXT: packuswb %xmm3, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm9 = [65535,65535,65535,0,0,0,65535,65535] +; SSE-NEXT: movdqa {{.*#+}} xmm8 = [65535,65535,65535,0,0,0,65535,65535] +; SSE-NEXT: movdqa %xmm8, %xmm9 ; SSE-NEXT: pandn %xmm0, %xmm9 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: pandn %xmm2, %xmm10 -; SSE-NEXT: pand %xmm4, %xmm11 -; SSE-NEXT: movdqa %xmm4, %xmm3 -; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: movdqa %xmm2, %xmm11 +; SSE-NEXT: pandn %xmm1, %xmm11 +; SSE-NEXT: pand %xmm4, %xmm10 +; SSE-NEXT: movdqa %xmm4, %xmm2 +; SSE-NEXT: pandn %xmm1, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm5, %xmm14 ; SSE-NEXT: pand %xmm4, %xmm14 -; SSE-NEXT: movdqa 80(%rdi), %xmm1 -; SSE-NEXT: movdqa %xmm1, %xmm13 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 80(%rdi), %xmm3 +; SSE-NEXT: movdqa %xmm3, %xmm13 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm4, %xmm13 -; SSE-NEXT: movdqa %xmm8, %xmm15 -; SSE-NEXT: pand %xmm4, %xmm8 +; SSE-NEXT: movdqa %xmm7, %xmm15 +; SSE-NEXT: pand %xmm4, %xmm7 ; SSE-NEXT: pand %xmm4, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm4, %xmm12 ; SSE-NEXT: movdqa %xmm4, %xmm2 ; SSE-NEXT: pandn %xmm5, %xmm4 ; SSE-NEXT: pand %xmm0, %xmm5 -; SSE-NEXT: por %xmm10, %xmm5 -; SSE-NEXT: movdqa %xmm5, %xmm10 -; SSE-NEXT: pand %xmm7, %xmm10 -; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,2,1,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,6,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,3,2,1,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm10[0,1,2,3,4,7,6,7] +; SSE-NEXT: por %xmm11, %xmm5 +; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm5[0,2,1,3] +; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm1, %xmm11 +; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,6,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,2,1,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[0,3,2,1,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm11[0,1,2,3,4,7,6,7] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 +; SSE-NEXT: pand %xmm8, %xmm0 ; SSE-NEXT: por %xmm9, %xmm0 -; SSE-NEXT: pandn %xmm1, %xmm12 -; SSE-NEXT: por %xmm12, %xmm11 -; SSE-NEXT: movdqa %xmm11, %xmm9 -; SSE-NEXT: pand %xmm7, %xmm9 -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[3,1,2,0] +; SSE-NEXT: pandn %xmm3, %xmm12 +; SSE-NEXT: por %xmm12, %xmm10 +; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm10[3,1,2,0] +; SSE-NEXT: pand %xmm1, %xmm9 +; SSE-NEXT: movdqa %xmm1, %xmm3 ; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,3,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,7,6,5] ; SSE-NEXT: packuswb %xmm9, %xmm9 -; SSE-NEXT: movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] -; SSE-NEXT: movdqa %xmm10, %xmm1 -; SSE-NEXT: pandn %xmm9, %xmm1 -; SSE-NEXT: pand %xmm10, %xmm0 -; SSE-NEXT: por %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{.*#+}} xmm11 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] +; SSE-NEXT: movdqa %xmm11, %xmm12 +; SSE-NEXT: pandn %xmm9, %xmm12 +; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: por %xmm0, %xmm12 ; SSE-NEXT: pxor %xmm9, %xmm9 ; SSE-NEXT: movdqa %xmm6, %xmm0 ; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15] @@ -568,15 +567,14 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pandn %xmm0, %xmm6 ; SSE-NEXT: por %xmm5, %xmm6 ; SSE-NEXT: packuswb %xmm6, %xmm6 -; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,0,0,0,65535,65535] -; SSE-NEXT: pand %xmm0, %xmm6 -; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: por %xmm0, %xmm6 -; SSE-NEXT: movdqa %xmm11, %xmm0 +; SSE-NEXT: pand %xmm8, %xmm6 +; SSE-NEXT: pandn %xmm1, %xmm8 +; SSE-NEXT: por %xmm8, %xmm6 +; SSE-NEXT: movdqa %xmm10, %xmm0 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5] -; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm9[8],xmm11[9],xmm9[9],xmm11[10],xmm9[10],xmm11[11],xmm9[11],xmm11[12],xmm9[12],xmm11[13],xmm9[13],xmm11[14],xmm9[14],xmm11[15],xmm9[15] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm11[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm10[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,7,6,4] ; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,65535,65535,0,65535,65535,0,65535] @@ -584,14 +582,14 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pandn %xmm0, %xmm5 ; SSE-NEXT: por %xmm1, %xmm5 ; SSE-NEXT: packuswb %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm10, %xmm12 -; SSE-NEXT: pandn %xmm0, %xmm12 -; SSE-NEXT: pand %xmm10, %xmm6 -; SSE-NEXT: por %xmm6, %xmm12 +; SSE-NEXT: movdqa %xmm11, %xmm10 +; SSE-NEXT: pandn %xmm0, %xmm10 +; SSE-NEXT: pand %xmm11, %xmm6 +; SSE-NEXT: por %xmm6, %xmm10 ; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm15 ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload ; SSE-NEXT: movdqa %xmm15, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm0 +; SSE-NEXT: pand %xmm3, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,3,4,5,6,7] @@ -599,11 +597,11 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,5,6] ; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: por %xmm3, %xmm14 -; SSE-NEXT: movdqa %xmm14, %xmm1 -; SSE-NEXT: pand %xmm7, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm14[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm3, %xmm1 +; SSE-NEXT: movdqa %xmm3, %xmm8 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] @@ -613,117 +611,118 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pandn %xmm1, %xmm5 ; SSE-NEXT: pand %xmm3, %xmm0 ; SSE-NEXT: por %xmm0, %xmm5 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload -; SSE-NEXT: pandn %xmm11, %xmm2 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload +; SSE-NEXT: pandn %xmm6, %xmm2 ; SSE-NEXT: por %xmm2, %xmm13 -; SSE-NEXT: movdqa %xmm13, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm13[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm8, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa %xmm10, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm10, %xmm5 -; SSE-NEXT: por %xmm5, %xmm1 +; SSE-NEXT: movdqa %xmm11, %xmm8 +; SSE-NEXT: pandn %xmm0, %xmm8 +; SSE-NEXT: pand %xmm11, %xmm5 +; SSE-NEXT: por %xmm5, %xmm8 ; SSE-NEXT: movdqa %xmm15, %xmm0 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7] ; SSE-NEXT: punpckhbw {{.*#+}} xmm15 = xmm15[8],xmm9[8],xmm15[9],xmm9[9],xmm15[10],xmm9[10],xmm15[11],xmm9[11],xmm15[12],xmm9[12],xmm15[13],xmm9[13],xmm15[14],xmm9[14],xmm15[15],xmm9[15] -; SSE-NEXT: movdqa %xmm15, %xmm2 -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[3,0] -; SSE-NEXT: movaps %xmm0, %xmm5 -; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm2[0,2] +; SSE-NEXT: movdqa %xmm15, %xmm1 +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0] +; SSE-NEXT: movaps %xmm0, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2] ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm15[0,0] ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm15[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm5[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm0, %xmm2 +; SSE-NEXT: packuswb %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm14, %xmm0 ; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7] ; SSE-NEXT: punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm9[8],xmm14[9],xmm9[9],xmm14[10],xmm9[10],xmm14[11],xmm9[11],xmm14[12],xmm9[12],xmm14[13],xmm9[13],xmm14[14],xmm9[14],xmm14[15],xmm9[15] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm14[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,7,7,7] -; SSE-NEXT: movdqa {{.*#+}} xmm6 = [0,65535,65535,0,65535,65535,65535,65535] -; SSE-NEXT: pand %xmm6, %xmm5 -; SSE-NEXT: pandn %xmm0, %xmm6 -; SSE-NEXT: por %xmm5, %xmm6 -; SSE-NEXT: pand %xmm3, %xmm2 -; SSE-NEXT: packuswb %xmm6, %xmm6 -; SSE-NEXT: pandn %xmm6, %xmm3 -; SSE-NEXT: por %xmm2, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm14[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7] +; SSE-NEXT: movdqa {{.*#+}} xmm5 = [0,65535,65535,0,65535,65535,65535,65535] +; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm5 +; SSE-NEXT: por %xmm2, %xmm5 +; SSE-NEXT: pand %xmm3, %xmm1 +; SSE-NEXT: packuswb %xmm5, %xmm5 +; SSE-NEXT: pandn %xmm5, %xmm3 +; SSE-NEXT: por %xmm1, %xmm3 ; SSE-NEXT: movdqa %xmm13, %xmm0 ; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,5] ; SSE-NEXT: punpcklbw {{.*#+}} xmm13 = xmm13[0],xmm9[0],xmm13[1],xmm9[1],xmm13[2],xmm9[2],xmm13[3],xmm9[3],xmm13[4],xmm9[4],xmm13[5],xmm9[5],xmm13[6],xmm9[6],xmm13[7],xmm9[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm13[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7] -; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,65535,65535,65535,0,65535,65535,0] -; SSE-NEXT: pand %xmm5, %xmm2 -; SSE-NEXT: pandn %xmm0, %xmm5 -; SSE-NEXT: por %xmm2, %xmm5 -; SSE-NEXT: pand %xmm10, %xmm3 -; SSE-NEXT: packuswb %xmm5, %xmm0 -; SSE-NEXT: pandn %xmm0, %xmm10 -; SSE-NEXT: por %xmm3, %xmm10 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm13[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,7,7] +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535,0,65535,65535,0] +; SSE-NEXT: pand %xmm2, %xmm1 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: por %xmm1, %xmm2 +; SSE-NEXT: pand %xmm11, %xmm3 +; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: pandn %xmm0, %xmm11 +; SSE-NEXT: por %xmm3, %xmm11 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm2, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2] -; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: packuswb %xmm1, %xmm0 ; SSE-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255] -; SSE-NEXT: movdqa %xmm3, %xmm2 -; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm3, %xmm1 +; SSE-NEXT: pandn %xmm0, %xmm1 ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm4[3,1,2,0] +; SSE-NEXT: pand %xmm2, %xmm0 +; SSE-NEXT: movdqa %xmm2, %xmm5 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm0[2,1,0,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm5, %xmm5 -; SSE-NEXT: pand %xmm3, %xmm5 -; SSE-NEXT: por %xmm2, %xmm5 +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm0[2,1,0,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: pand %xmm3, %xmm2 +; SSE-NEXT: por %xmm1, %xmm2 ; SSE-NEXT: movdqa {{.*#+}} xmm13 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: pand %xmm13, %xmm11 +; SSE-NEXT: movdqa %xmm6, %xmm1 +; SSE-NEXT: pand %xmm13, %xmm1 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,65535,65535,0,0,0] -; SSE-NEXT: pand %xmm0, %xmm5 -; SSE-NEXT: por %xmm11, %xmm13 -; SSE-NEXT: pand %xmm13, %xmm7 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm7[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,1,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: por %xmm1, %xmm13 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm13[0,2,1,3] +; SSE-NEXT: pand %xmm5, %xmm1 +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7] +; SSE-NEXT: packuswb %xmm1, %xmm1 ; SSE-NEXT: movdqa %xmm0, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: por %xmm5, %xmm6 -; SSE-NEXT: movdqa %xmm8, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm9[8],xmm2[9],xmm9[9],xmm2[10],xmm9[10],xmm2[11],xmm9[11],xmm2[12],xmm9[12],xmm2[13],xmm9[13],xmm2[14],xmm9[14],xmm2[15],xmm9[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3],xmm8[4],xmm9[4],xmm8[5],xmm9[5],xmm8[6],xmm9[6],xmm8[7],xmm9[7] -; SSE-NEXT: movdqa %xmm8, %xmm5 -; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[1,0],xmm2[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,0],xmm2[2,3] -; SSE-NEXT: psrlq $48, %xmm2 -; SSE-NEXT: psrldq {{.*#+}} xmm5 = xmm5[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm8[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm2[0,1,2,3,4,4,5,7] -; SSE-NEXT: packuswb %xmm5, %xmm7 +; SSE-NEXT: pandn %xmm1, %xmm6 +; SSE-NEXT: por %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm7, %xmm1 +; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm9[8],xmm1[9],xmm9[9],xmm1[10],xmm9[10],xmm1[11],xmm9[11],xmm1[12],xmm9[12],xmm1[13],xmm9[13],xmm1[14],xmm9[14],xmm1[15],xmm9[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm9[0],xmm7[1],xmm9[1],xmm7[2],xmm9[2],xmm7[3],xmm9[3],xmm7[4],xmm9[4],xmm7[5],xmm9[5],xmm7[6],xmm9[6],xmm7[7],xmm9[7] +; SSE-NEXT: movdqa %xmm7, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[1,0],xmm1[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,0],xmm1[2,3] +; SSE-NEXT: psrlq $48, %xmm1 +; SSE-NEXT: psrldq {{.*#+}} xmm2 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm7[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm2, %xmm1 ; SSE-NEXT: movdqa %xmm4, %xmm2 ; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm9[8],xmm2[9],xmm9[9],xmm2[10],xmm9[10],xmm2[11],xmm9[11],xmm2[12],xmm9[12],xmm2[13],xmm9[13],xmm2[14],xmm9[14],xmm2[15],xmm9[15] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3] @@ -738,29 +737,28 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: por %xmm4, %xmm2 ; SSE-NEXT: packuswb %xmm2, %xmm2 ; SSE-NEXT: pand %xmm3, %xmm2 -; SSE-NEXT: pandn %xmm7, %xmm3 +; SSE-NEXT: pandn %xmm1, %xmm3 ; SSE-NEXT: por %xmm3, %xmm2 -; SSE-NEXT: movdqa %xmm13, %xmm3 -; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm9[0],xmm3[1],xmm9[1],xmm3[2],xmm9[2],xmm3[3],xmm9[3],xmm3[4],xmm9[4],xmm3[5],xmm9[5],xmm3[6],xmm9[6],xmm3[7],xmm9[7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,1,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,1,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm13, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3],xmm1[4],xmm9[4],xmm1[5],xmm9[5],xmm1[6],xmm9[6],xmm1[7],xmm9[7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,1,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7] ; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm9[8],xmm13[9],xmm9[9],xmm13[10],xmm9[10],xmm13[11],xmm9[11],xmm13[12],xmm9[12],xmm13[13],xmm9[13],xmm13[14],xmm9[14],xmm13[15],xmm9[15] -; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,65535,0,65535,0,0] -; SSE-NEXT: pand %xmm4, %xmm3 -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm13[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,7,4] -; SSE-NEXT: pandn %xmm5, %xmm4 -; SSE-NEXT: por %xmm3, %xmm4 +; SSE-NEXT: movdqa {{.*#+}} xmm3 = [65535,65535,65535,65535,0,65535,0,0] +; SSE-NEXT: pand %xmm3, %xmm1 +; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm13[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,7,4] +; SSE-NEXT: pandn %xmm4, %xmm3 +; SSE-NEXT: por %xmm1, %xmm3 ; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: packuswb %xmm4, %xmm3 -; SSE-NEXT: pandn %xmm3, %xmm0 +; SSE-NEXT: packuswb %xmm3, %xmm1 +; SSE-NEXT: pandn %xmm1, %xmm0 ; SSE-NEXT: por %xmm2, %xmm0 -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload -; SSE-NEXT: movaps %xmm2, (%rsi) -; SSE-NEXT: movdqa %xmm12, (%rdx) -; SSE-NEXT: movdqa %xmm1, (%rcx) -; SSE-NEXT: movdqa %xmm10, (%r8) +; SSE-NEXT: movdqa %xmm12, (%rsi) +; SSE-NEXT: movdqa %xmm10, (%rdx) +; SSE-NEXT: movdqa %xmm8, (%rcx) +; SSE-NEXT: movdqa %xmm11, (%r8) ; SSE-NEXT: movdqa %xmm6, (%r9) ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax ; SSE-NEXT: movdqa %xmm0, (%rax) @@ -1076,27 +1074,27 @@ define void @load_i8_stride6_vf16(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5) nounwind { ; SSE-LABEL: load_i8_stride6_vf32: ; SSE: # %bb.0: -; SSE-NEXT: subq $280, %rsp # imm = 0x118 -; SSE-NEXT: movdqa 64(%rdi), %xmm10 -; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: subq $264, %rsp # imm = 0x108 +; SSE-NEXT: movdqa 64(%rdi), %xmm7 +; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 80(%rdi), %xmm9 -; SSE-NEXT: movdqa (%rdi), %xmm14 -; SSE-NEXT: movdqa 16(%rdi), %xmm2 +; SSE-NEXT: movdqa (%rdi), %xmm12 +; SSE-NEXT: movdqa 16(%rdi), %xmm14 ; SSE-NEXT: movdqa 32(%rdi), %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 48(%rdi), %xmm5 -; SSE-NEXT: movdqa {{.*#+}} xmm7 = [65535,65535,0,65535,65535,0,65535,65535] -; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm10 = [65535,65535,0,65535,65535,0,65535,65535] +; SSE-NEXT: movdqa %xmm10, %xmm0 ; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: movdqa %xmm12, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm11 = [65535,0,65535,65535,0,65535,65535,0] +; SSE-NEXT: movdqa %xmm11, %xmm1 ; SSE-NEXT: pandn %xmm5, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm1 +; SSE-NEXT: movdqa %xmm10, %xmm1 ; SSE-NEXT: pandn %xmm5, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm5, %xmm15 -; SSE-NEXT: pand %xmm7, %xmm15 +; SSE-NEXT: pand %xmm10, %xmm15 ; SSE-NEXT: por %xmm0, %xmm15 ; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] ; SSE-NEXT: movdqa %xmm15, %xmm0 @@ -1107,238 +1105,237 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] ; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,0,0,0,65535,65535] -; SSE-NEXT: movdqa %xmm12, %xmm1 -; SSE-NEXT: pandn %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm14, %xmm11 -; SSE-NEXT: pand %xmm12, %xmm11 -; SSE-NEXT: por %xmm1, %xmm11 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,0,0,0,65535,65535] ; SSE-NEXT: movdqa %xmm11, %xmm1 +; SSE-NEXT: pandn %xmm14, %xmm1 +; SSE-NEXT: movdqa %xmm12, %xmm8 +; SSE-NEXT: pand %xmm11, %xmm8 +; SSE-NEXT: por %xmm1, %xmm8 +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm8[0,2,1,3] ; SSE-NEXT: pand %xmm3, %xmm1 ; SSE-NEXT: movdqa %xmm3, %xmm6 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,7] ; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: pand %xmm4, %xmm1 -; SSE-NEXT: movdqa %xmm4, %xmm3 +; SSE-NEXT: pand %xmm2, %xmm1 +; SSE-NEXT: movdqa %xmm2, %xmm3 +; SSE-NEXT: movdqa %xmm2, %xmm5 ; SSE-NEXT: pandn %xmm0, %xmm3 ; SSE-NEXT: por %xmm3, %xmm1 -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: pandn %xmm9, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm10 -; SSE-NEXT: por %xmm0, %xmm10 ; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: pandn %xmm9, %xmm0 +; SSE-NEXT: pand %xmm10, %xmm7 +; SSE-NEXT: por %xmm0, %xmm7 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm7[3,1,2,0] ; SSE-NEXT: pand %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm6, %xmm8 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] -; SSE-NEXT: movdqa %xmm5, %xmm3 -; SSE-NEXT: pandn %xmm0, %xmm3 -; SSE-NEXT: pand %xmm5, %xmm1 -; SSE-NEXT: por %xmm1, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 128(%rdi), %xmm13 -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: pandn %xmm13, %xmm0 -; SSE-NEXT: movdqa 144(%rdi), %xmm6 -; SSE-NEXT: movdqa %xmm12, %xmm3 -; SSE-NEXT: pandn %xmm6, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm3 -; SSE-NEXT: pandn %xmm6, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm7, %xmm6 -; SSE-NEXT: por %xmm0, %xmm6 -; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm0[0,3,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,2,3] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] -; SSE-NEXT: packuswb %xmm3, %xmm0 -; SSE-NEXT: movdqa %xmm4, %xmm8 -; SSE-NEXT: pandn %xmm0, %xmm8 -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: pandn %xmm14, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 112(%rdi), %xmm1 -; SSE-NEXT: movdqa %xmm12, %xmm4 -; SSE-NEXT: pandn %xmm1, %xmm4 -; SSE-NEXT: movdqa 160(%rdi), %xmm7 -; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm7 -; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: pandn %xmm2, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm14 -; SSE-NEXT: pandn %xmm9, %xmm12 -; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm9 -; SSE-NEXT: movdqa %xmm9, (%rsp) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: pandn %xmm1, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 96(%rdi), %xmm3 -; SSE-NEXT: movdqa %xmm3, %xmm5 -; SSE-NEXT: pand %xmm0, %xmm5 -; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 176(%rdi), %xmm5 -; SSE-NEXT: movdqa %xmm5, %xmm12 -; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm12 -; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; SSE-NEXT: movdqa %xmm9, %xmm12 -; SSE-NEXT: pand %xmm0, %xmm9 -; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm13, %xmm2 -; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] +; SSE-NEXT: movdqa %xmm3, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pand %xmm3, %xmm1 +; SSE-NEXT: por %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, %xmm2 -; SSE-NEXT: pand %xmm0, %xmm1 +; SSE-NEXT: movdqa 128(%rdi), %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pandn %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: pandn %xmm1, %xmm0 +; SSE-NEXT: movdqa 144(%rdi), %xmm1 +; SSE-NEXT: movdqa %xmm11, %xmm2 +; SSE-NEXT: pandn %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm3, %xmm2 -; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2 -; SSE-NEXT: por %xmm4, %xmm2 -; SSE-NEXT: movdqa %xmm2, %xmm4 -; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: movdqa %xmm10, %xmm2 +; SSE-NEXT: pandn %xmm1, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: pand %xmm10, %xmm2 +; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, %xmm0 +; SSE-NEXT: pand %xmm6, %xmm0 +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm0[0,3,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,2,3] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] +; SSE-NEXT: packuswb %xmm3, %xmm0 +; SSE-NEXT: movdqa %xmm5, %xmm6 +; SSE-NEXT: pandn %xmm0, %xmm6 +; SSE-NEXT: movdqa %xmm10, %xmm1 +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: pandn %xmm12, %xmm0 +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 112(%rdi), %xmm0 +; SSE-NEXT: movdqa %xmm11, %xmm3 +; SSE-NEXT: pandn %xmm0, %xmm3 +; SSE-NEXT: movdqa 160(%rdi), %xmm5 +; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm10, %xmm5 +; SSE-NEXT: movdqa %xmm10, %xmm4 +; SSE-NEXT: pandn %xmm14, %xmm4 +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm10, %xmm12 +; SSE-NEXT: movdqa %xmm11, %xmm4 +; SSE-NEXT: pandn %xmm9, %xmm4 +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm9, %xmm11 +; SSE-NEXT: pand %xmm1, %xmm11 +; SSE-NEXT: movdqa %xmm1, %xmm4 +; SSE-NEXT: pandn %xmm0, %xmm4 +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 96(%rdi), %xmm13 +; SSE-NEXT: movdqa %xmm13, %xmm4 ; SSE-NEXT: pand %xmm1, %xmm4 -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,1,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,6,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,3,2,1,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,7] -; SSE-NEXT: packuswb %xmm4, %xmm4 -; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4 -; SSE-NEXT: por %xmm8, %xmm4 -; SSE-NEXT: pandn %xmm5, %xmm0 -; SSE-NEXT: por %xmm0, %xmm7 -; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 176(%rdi), %xmm4 +; SSE-NEXT: movdqa %xmm4, %xmm10 +; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm1, %xmm10 +; SSE-NEXT: movdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload +; SSE-NEXT: movdqa %xmm9, %xmm10 +; SSE-NEXT: pand %xmm1, %xmm9 +; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm1, %xmm14 +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload +; SSE-NEXT: movdqa %xmm14, %xmm9 +; SSE-NEXT: pand %xmm1, %xmm14 +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm1, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5] -; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm3 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] -; SSE-NEXT: movdqa %xmm3, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm3, %xmm4 -; SSE-NEXT: por %xmm4, %xmm1 +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm1, %xmm14 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pxor %xmm9, %xmm9 -; SSE-NEXT: movdqa %xmm15, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3],xmm15[4],xmm9[4],xmm15[5],xmm9[5],xmm15[6],xmm9[6],xmm15[7],xmm9[7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm15[2,2,3,3] -; SSE-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3] -; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill +; SSE-NEXT: pandn %xmm13, %xmm1 +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm13, %xmm1 +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1 +; SSE-NEXT: por %xmm3, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,1,3] +; SSE-NEXT: movdqa {{.*#+}} xmm0 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm0, %xmm3 +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,3,2,1,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm3 +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 +; SSE-NEXT: por %xmm6, %xmm3 +; SSE-NEXT: pandn %xmm4, %xmm14 +; SSE-NEXT: por %xmm14, %xmm5 +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm5[3,1,2,0] +; SSE-NEXT: pand %xmm0, %xmm4 +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,4,7,6,5] +; SSE-NEXT: packuswb %xmm4, %xmm4 +; SSE-NEXT: movdqa {{.*#+}} xmm13 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: pandn %xmm4, %xmm0 +; SSE-NEXT: pand %xmm13, %xmm3 +; SSE-NEXT: por %xmm3, %xmm0 +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pxor %xmm4, %xmm4 +; SSE-NEXT: movdqa %xmm15, %xmm3 +; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm4[0],xmm15[1],xmm4[1],xmm15[2],xmm4[2],xmm15[3],xmm4[3],xmm15[4],xmm4[4],xmm15[5],xmm4[5],xmm15[6],xmm4[6],xmm15[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm15[2,2,3,3] +; SSE-NEXT: punpcklwd {{.*#+}} xmm14 = xmm14[0],xmm3[0],xmm14[1],xmm3[1],xmm14[2],xmm3[2],xmm14[3],xmm3[3] +; SSE-NEXT: psrld $16, %xmm3 ; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm15[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,5,7,6,7] -; SSE-NEXT: punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm0[2],xmm15[3],xmm0[3] -; SSE-NEXT: packuswb %xmm15, %xmm5 -; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,0,0,0,65535,65535] -; SSE-NEXT: movdqa %xmm4, %xmm1 -; SSE-NEXT: pandn %xmm5, %xmm1 -; SSE-NEXT: movdqa %xmm11, %xmm5 -; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm9[8],xmm5[9],xmm9[9],xmm5[10],xmm9[10],xmm5[11],xmm9[11],xmm5[12],xmm9[12],xmm5[13],xmm9[13],xmm5[14],xmm9[14],xmm5[15],xmm9[15] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,1,1,1,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,7,6,7] +; SSE-NEXT: punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm3[2],xmm15[3],xmm3[3] +; SSE-NEXT: packuswb %xmm15, %xmm14 +; SSE-NEXT: movdqa {{.*#+}} xmm6 = [65535,65535,65535,0,0,0,65535,65535] +; SSE-NEXT: movdqa %xmm6, %xmm3 +; SSE-NEXT: pandn %xmm14, %xmm3 +; SSE-NEXT: movdqa %xmm8, %xmm14 +; SSE-NEXT: punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm4[8],xmm14[9],xmm4[9],xmm14[10],xmm4[10],xmm14[11],xmm4[11],xmm14[12],xmm4[12],xmm14[13],xmm4[13],xmm14[14],xmm4[14],xmm14[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm14[2,1,0,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm14[1,1,1,1,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,5,7,6,7] ; SSE-NEXT: movdqa {{.*#+}} xmm15 = [65535,65535,0,65535,0,0,65535,65535] ; SSE-NEXT: movdqa %xmm15, %xmm0 -; SSE-NEXT: pandn %xmm5, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm9[0],xmm11[1],xmm9[1],xmm11[2],xmm9[2],xmm11[3],xmm9[3],xmm11[4],xmm9[4],xmm11[5],xmm9[5],xmm11[6],xmm9[6],xmm11[7],xmm9[7] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm11[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[1,3,2,0,4,5,6,7] -; SSE-NEXT: pand %xmm15, %xmm5 -; SSE-NEXT: por %xmm0, %xmm5 -; SSE-NEXT: packuswb %xmm5, %xmm5 -; SSE-NEXT: pand %xmm4, %xmm5 -; SSE-NEXT: por %xmm1, %xmm5 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7] +; SSE-NEXT: pandn %xmm14, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3],xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7] +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm8[1,3,2,0,4,5,6,7] +; SSE-NEXT: pand %xmm15, %xmm14 +; SSE-NEXT: por %xmm0, %xmm14 +; SSE-NEXT: packuswb %xmm14, %xmm14 +; SSE-NEXT: pand %xmm6, %xmm14 +; SSE-NEXT: por %xmm3, %xmm14 +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5] ; SSE-NEXT: movdqa {{.*#+}} xmm8 = [65535,65535,65535,0,65535,65535,0,65535] -; SSE-NEXT: movdqa %xmm8, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm10[3,1,2,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm8, %xmm3 +; SSE-NEXT: pandn %xmm0, %xmm3 +; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm4[8],xmm7[9],xmm4[9],xmm7[10],xmm4[10],xmm7[11],xmm4[11],xmm7[12],xmm4[12],xmm7[13],xmm4[13],xmm7[14],xmm4[14],xmm7[15],xmm4[15] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm7[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,7,6,4] ; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: por %xmm1, %xmm0 +; SSE-NEXT: por %xmm3, %xmm0 ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa %xmm3, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm3, %xmm5 -; SSE-NEXT: por %xmm5, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1],xmm6[2],xmm9[2],xmm6[3],xmm9[3],xmm6[4],xmm9[4],xmm6[5],xmm9[5],xmm6[6],xmm9[6],xmm6[7],xmm9[7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm6[2,2,3,3] -; SSE-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3] -; SSE-NEXT: psrld $16, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm6[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,7,6,7] -; SSE-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm0[2],xmm5[3],xmm0[3] -; SSE-NEXT: packuswb %xmm5, %xmm1 +; SSE-NEXT: movdqa %xmm13, %xmm3 +; SSE-NEXT: pandn %xmm0, %xmm3 +; SSE-NEXT: pand %xmm13, %xmm14 +; SSE-NEXT: por %xmm14, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm2, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm9[8],xmm0[9],xmm9[9],xmm0[10],xmm9[10],xmm0[11],xmm9[11],xmm0[12],xmm9[12],xmm0[13],xmm9[13],xmm0[14],xmm9[14],xmm0[15],xmm9[15] +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,2,3,3] +; SSE-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3] +; SSE-NEXT: psrld $16, %xmm0 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,7,6,7] +; SSE-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm0[2],xmm2[3],xmm0[3] +; SSE-NEXT: packuswb %xmm2, %xmm3 +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,1,1,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,7,6,7] -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm9[0],xmm2[1],xmm9[1],xmm2[2],xmm9[2],xmm2[3],xmm9[3],xmm2[4],xmm9[4],xmm2[5],xmm9[5],xmm2[6],xmm9[6],xmm2[7],xmm9[7] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,3,2,0,4,5,6,7] -; SSE-NEXT: pand %xmm15, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,3,2,0,4,5,6,7] +; SSE-NEXT: pand %xmm15, %xmm1 ; SSE-NEXT: pandn %xmm0, %xmm15 -; SSE-NEXT: por %xmm2, %xmm15 +; SSE-NEXT: por %xmm1, %xmm15 ; SSE-NEXT: packuswb %xmm15, %xmm15 -; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: pand %xmm4, %xmm15 -; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: por %xmm0, %xmm15 -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3],xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7] +; SSE-NEXT: pand %xmm6, %xmm15 +; SSE-NEXT: pandn %xmm3, %xmm6 +; SSE-NEXT: por %xmm6, %xmm15 +; SSE-NEXT: movdqa %xmm5, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3],xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,5,5,5] -; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm9[8],xmm7[9],xmm9[9],xmm7[10],xmm9[10],xmm7[11],xmm9[11],xmm7[12],xmm9[12],xmm7[13],xmm9[13],xmm7[14],xmm9[14],xmm7[15],xmm9[15] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm7[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm5[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,7,6,4] ; SSE-NEXT: pand %xmm8, %xmm1 ; SSE-NEXT: pandn %xmm0, %xmm8 ; SSE-NEXT: por %xmm1, %xmm8 ; SSE-NEXT: packuswb %xmm8, %xmm0 -; SSE-NEXT: movdqa %xmm3, %xmm1 +; SSE-NEXT: movdqa %xmm13, %xmm1 ; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm3, %xmm15 -; SSE-NEXT: movdqa %xmm3, %xmm8 +; SSE-NEXT: pand %xmm13, %xmm15 +; SSE-NEXT: movdqa %xmm13, %xmm7 ; SSE-NEXT: por %xmm15, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: pand %xmm4, %xmm12 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm12, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm11 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,0,65535,65535,0,65535,65535,0] +; SSE-NEXT: pand %xmm5, %xmm10 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm15 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm15, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,3,4,5,6,7] @@ -1346,11 +1343,10 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,4,5,6] ; SSE-NEXT: packuswb %xmm1, %xmm2 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7] +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm12[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm15, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[1,2,3,0,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,5,5,5,5] @@ -1360,27 +1356,26 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pandn %xmm1, %xmm3 ; SSE-NEXT: pand %xmm0, %xmm2 ; SSE-NEXT: por %xmm2, %xmm3 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload -; SSE-NEXT: movdqa (%rsp), %xmm15 # 16-byte Reload -; SSE-NEXT: por %xmm1, %xmm15 -; SSE-NEXT: movdqa %xmm15, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7] +; SSE-NEXT: pandn %xmm14, %xmm1 +; SSE-NEXT: por %xmm1, %xmm11 +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm11[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm15, %xmm1 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm8, %xmm2 +; SSE-NEXT: movdqa %xmm13, %xmm2 ; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm8, %xmm3 +; SSE-NEXT: pand %xmm13, %xmm3 ; SSE-NEXT: por %xmm3, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm4, %xmm13 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm13, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 +; SSE-NEXT: pand %xmm5, %xmm9 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm9, %xmm1 +; SSE-NEXT: pand %xmm15, %xmm1 ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,3,4,5,6,7] @@ -1388,12 +1383,11 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,4,5,6] ; SSE-NEXT: packuswb %xmm2, %xmm1 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm4, %xmm2 -; SSE-NEXT: pand %xmm11, %xmm2 -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm13[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm15, %xmm2 ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5] @@ -1402,294 +1396,289 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pandn %xmm2, %xmm3 ; SSE-NEXT: pand %xmm0, %xmm1 ; SSE-NEXT: por %xmm1, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; SSE-NEXT: por %xmm1, %xmm10 -; SSE-NEXT: movdqa %xmm10, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload +; SSE-NEXT: por %xmm1, %xmm8 +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm8[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm15, %xmm1 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm8, %xmm2 +; SSE-NEXT: movdqa %xmm7, %xmm2 ; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm8, %xmm3 +; SSE-NEXT: pand %xmm7, %xmm3 ; SSE-NEXT: por %xmm3, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm12, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3],xmm1[4],xmm9[4],xmm1[5],xmm9[5],xmm1[6],xmm9[6],xmm1[7],xmm9[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm9[8],xmm12[9],xmm9[9],xmm12[10],xmm9[10],xmm12[11],xmm9[11],xmm12[12],xmm9[12],xmm12[13],xmm9[13],xmm12[14],xmm9[14],xmm12[15],xmm9[15] -; SSE-NEXT: movdqa %xmm12, %xmm3 -; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm1[3,0] -; SSE-NEXT: movaps %xmm1, %xmm5 -; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,1],xmm3[0,2] -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm12[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm12[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm5[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,2] +; SSE-NEXT: movdqa %xmm10, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm4[8],xmm10[9],xmm4[9],xmm10[10],xmm4[10],xmm10[11],xmm4[11],xmm10[12],xmm4[12],xmm10[13],xmm4[13],xmm10[14],xmm4[14],xmm10[15],xmm4[15] +; SSE-NEXT: movdqa %xmm10, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm1[3,0] +; SSE-NEXT: movaps %xmm1, %xmm3 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[0,2] +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm10[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm10[2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm1, %xmm3 -; SSE-NEXT: movdqa %xmm14, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm9[0],xmm1[1],xmm9[1],xmm1[2],xmm9[2],xmm1[3],xmm9[3],xmm1[4],xmm9[4],xmm1[5],xmm9[5],xmm1[6],xmm9[6],xmm1[7],xmm9[7] +; SSE-NEXT: packuswb %xmm1, %xmm2 +; SSE-NEXT: movdqa %xmm12, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm1[3,1,2,1,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm1[3,1,2,1,4,5,6,7] ; SSE-NEXT: movdqa {{.*#+}} xmm1 = [0,65535,65535,0,65535,65535,65535,65535] -; SSE-NEXT: movdqa %xmm1, %xmm6 -; SSE-NEXT: pandn %xmm5, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm9[8],xmm14[9],xmm9[9],xmm14[10],xmm9[10],xmm14[11],xmm9[11],xmm14[12],xmm9[12],xmm14[13],xmm9[13],xmm14[14],xmm9[14],xmm14[15],xmm9[15] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm14[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,7,7,7,7] -; SSE-NEXT: pand %xmm1, %xmm5 -; SSE-NEXT: por %xmm6, %xmm5 -; SSE-NEXT: packuswb %xmm5, %xmm5 -; SSE-NEXT: movdqa %xmm0, %xmm6 -; SSE-NEXT: pandn %xmm5, %xmm6 -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: por %xmm3, %xmm6 -; SSE-NEXT: movdqa %xmm15, %xmm3 -; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm9[8],xmm3[9],xmm9[9],xmm3[10],xmm9[10],xmm3[11],xmm9[11],xmm3[12],xmm9[12],xmm3[13],xmm9[13],xmm3[14],xmm9[14],xmm3[15],xmm9[15] -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,2,1] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm3[0,1,2,3,7,5,6,5] -; SSE-NEXT: movdqa {{.*#+}} xmm3 = [65535,65535,65535,65535,0,65535,65535,0] -; SSE-NEXT: movdqa %xmm3, %xmm7 -; SSE-NEXT: pandn %xmm5, %xmm7 -; SSE-NEXT: punpcklbw {{.*#+}} xmm15 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3],xmm15[4],xmm9[4],xmm15[5],xmm9[5],xmm15[6],xmm9[6],xmm15[7],xmm9[7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm15[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,7] -; SSE-NEXT: pand %xmm3, %xmm5 -; SSE-NEXT: por %xmm7, %xmm5 -; SSE-NEXT: packuswb %xmm5, %xmm5 -; SSE-NEXT: movdqa %xmm8, %xmm15 -; SSE-NEXT: pandn %xmm5, %xmm15 -; SSE-NEXT: pand %xmm8, %xmm6 -; SSE-NEXT: por %xmm6, %xmm15 -; SSE-NEXT: movdqa %xmm13, %xmm5 -; SSE-NEXT: pxor %xmm2, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm2[8],xmm13[9],xmm2[9],xmm13[10],xmm2[10],xmm13[11],xmm2[11],xmm13[12],xmm2[12],xmm13[13],xmm2[13],xmm13[14],xmm2[14],xmm13[15],xmm2[15] -; SSE-NEXT: movdqa %xmm13, %xmm6 -; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,0],xmm5[3,0] -; SSE-NEXT: movaps %xmm5, %xmm7 -; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm6[0,2] -; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[1,0],xmm13[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm13[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm7[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,2] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm5, %xmm6 -; SSE-NEXT: movdqa %xmm4, %xmm5 -; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[3,1,2,1,4,5,6,7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15] -; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm4[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,7,7,7] -; SSE-NEXT: pand %xmm1, %xmm7 -; SSE-NEXT: pandn %xmm5, %xmm1 -; SSE-NEXT: por %xmm7, %xmm1 -; SSE-NEXT: pand %xmm0, %xmm6 +; SSE-NEXT: movdqa %xmm1, %xmm5 +; SSE-NEXT: pandn %xmm3, %xmm5 +; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm4[8],xmm12[9],xmm4[9],xmm12[10],xmm4[10],xmm12[11],xmm4[11],xmm12[12],xmm4[12],xmm12[13],xmm4[13],xmm12[14],xmm4[14],xmm12[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm12[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,7,7,7,7] +; SSE-NEXT: pand %xmm1, %xmm3 +; SSE-NEXT: por %xmm5, %xmm3 +; SSE-NEXT: packuswb %xmm3, %xmm3 +; SSE-NEXT: movdqa %xmm0, %xmm5 +; SSE-NEXT: pandn %xmm3, %xmm5 +; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: por %xmm2, %xmm5 +; SSE-NEXT: movdqa %xmm11, %xmm2 +; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm2[0,1,2,3,7,5,6,5] +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,65535,0,65535,65535,0] +; SSE-NEXT: movdqa %xmm2, %xmm6 +; SSE-NEXT: pandn %xmm3, %xmm6 +; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3],xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm11[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,7,7] +; SSE-NEXT: pand %xmm2, %xmm3 +; SSE-NEXT: por %xmm6, %xmm3 +; SSE-NEXT: packuswb %xmm3, %xmm3 +; SSE-NEXT: movdqa %xmm7, %xmm6 +; SSE-NEXT: pandn %xmm3, %xmm6 +; SSE-NEXT: pand %xmm7, %xmm5 +; SSE-NEXT: por %xmm5, %xmm6 +; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm9, %xmm3 +; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15] +; SSE-NEXT: movdqa %xmm9, %xmm5 +; SSE-NEXT: shufps {{.*#+}} xmm5 = xmm5[2,0],xmm3[3,0] +; SSE-NEXT: movaps %xmm3, %xmm6 +; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm5[0,2] +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm9[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm9[2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm6[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,0,2] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,3,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm3, %xmm5 +; SSE-NEXT: movdqa %xmm13, %xmm3 +; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[3,1,2,1,4,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm4[8],xmm13[9],xmm4[9],xmm13[10],xmm4[10],xmm13[11],xmm4[11],xmm13[12],xmm4[12],xmm13[13],xmm4[13],xmm13[14],xmm4[14],xmm13[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm13[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,7,7,7] +; SSE-NEXT: pand %xmm1, %xmm6 +; SSE-NEXT: pandn %xmm3, %xmm1 +; SSE-NEXT: por %xmm6, %xmm1 +; SSE-NEXT: pand %xmm0, %xmm5 ; SSE-NEXT: packuswb %xmm1, %xmm1 ; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: por %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm10, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] +; SSE-NEXT: por %xmm5, %xmm0 +; SSE-NEXT: movdqa %xmm8, %xmm1 +; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,7,5,6,5] -; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3],xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm10[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,5,7,7] -; SSE-NEXT: pand %xmm3, %xmm5 -; SSE-NEXT: pandn %xmm1, %xmm3 -; SSE-NEXT: por %xmm5, %xmm3 -; SSE-NEXT: movdqa %xmm8, %xmm4 -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: packuswb %xmm3, %xmm1 -; SSE-NEXT: pandn %xmm1, %xmm4 -; SSE-NEXT: por %xmm0, %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm9, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm4[0],xmm8[1],xmm4[1],xmm8[2],xmm4[2],xmm8[3],xmm4[3],xmm8[4],xmm4[4],xmm8[5],xmm4[5],xmm8[6],xmm4[6],xmm8[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm8[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,7,7] +; SSE-NEXT: pand %xmm2, %xmm3 +; SSE-NEXT: pandn %xmm1, %xmm2 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm7, %xmm13 +; SSE-NEXT: pand %xmm7, %xmm0 +; SSE-NEXT: packuswb %xmm2, %xmm1 +; SSE-NEXT: pandn %xmm1, %xmm13 +; SSE-NEXT: por %xmm0, %xmm13 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: pand %xmm15, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2] ; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm5 = [255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255] -; SSE-NEXT: movdqa %xmm5, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm2 = [255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255] +; SSE-NEXT: movdqa %xmm2, %xmm1 ; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm9[3,1,2,0] +; SSE-NEXT: pand %xmm15, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm0[2,1,0,3,4,5,6,7] ; SSE-NEXT: packuswb %xmm3, %xmm3 -; SSE-NEXT: pand %xmm5, %xmm3 +; SSE-NEXT: pand %xmm2, %xmm3 ; SSE-NEXT: por %xmm1, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload -; SSE-NEXT: movdqa {{.*#+}} xmm13 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: pand %xmm13, %xmm12 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm12, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSE-NEXT: movdqa %xmm14, %xmm11 +; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,0,65535,65535,0,65535,65535,0] +; SSE-NEXT: pand %xmm12, %xmm11 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm11[0,2,1,3] +; SSE-NEXT: pand %xmm15, %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm0, %xmm6 +; SSE-NEXT: packuswb %xmm0, %xmm5 ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,65535,65535,65535,0,0,0] ; SSE-NEXT: movdqa %xmm0, %xmm8 -; SSE-NEXT: pandn %xmm6, %xmm8 +; SSE-NEXT: pandn %xmm5, %xmm8 ; SSE-NEXT: pand %xmm0, %xmm3 ; SSE-NEXT: por %xmm3, %xmm8 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload ; SSE-NEXT: movdqa %xmm14, %xmm3 -; SSE-NEXT: pand %xmm11, %xmm3 -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm3[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[2,1,2,3,4,5,6,7] +; SSE-NEXT: pand %xmm15, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm3[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,2] -; SSE-NEXT: packuswb %xmm6, %xmm3 -; SSE-NEXT: movdqa %xmm5, %xmm6 -; SSE-NEXT: pandn %xmm3, %xmm6 +; SSE-NEXT: packuswb %xmm5, %xmm3 +; SSE-NEXT: movdqa %xmm2, %xmm5 +; SSE-NEXT: pandn %xmm3, %xmm5 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm1, %xmm3 -; SSE-NEXT: pand %xmm11, %xmm3 -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,1,2,0] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[3,1,2,0] +; SSE-NEXT: pand %xmm15, %xmm3 ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm3[2,1,0,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm7, %xmm7 -; SSE-NEXT: pand %xmm5, %xmm7 -; SSE-NEXT: por %xmm6, %xmm7 +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm3[2,1,0,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm6, %xmm6 +; SSE-NEXT: pand %xmm2, %xmm6 +; SSE-NEXT: por %xmm5, %xmm6 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: pand %xmm13, %xmm3 -; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload -; SSE-NEXT: por %xmm3, %xmm13 -; SSE-NEXT: movdqa %xmm11, %xmm3 -; SSE-NEXT: pand %xmm13, %xmm3 -; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3] +; SSE-NEXT: pand %xmm12, %xmm3 +; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload +; SSE-NEXT: por %xmm3, %xmm12 +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm12[0,2,1,3] +; SSE-NEXT: pand %xmm15, %xmm3 ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,2,1,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm3[0,1,0,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm3, %xmm6 +; SSE-NEXT: packuswb %xmm3, %xmm5 ; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: pandn %xmm6, %xmm3 -; SSE-NEXT: pand %xmm0, %xmm7 -; SSE-NEXT: por %xmm7, %xmm3 -; SSE-NEXT: movdqa %xmm9, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm2[0],xmm9[1],xmm2[1],xmm9[2],xmm2[2],xmm9[3],xmm2[3],xmm9[4],xmm2[4],xmm9[5],xmm2[5],xmm9[6],xmm2[6],xmm9[7],xmm2[7] -; SSE-NEXT: movdqa %xmm9, %xmm7 -; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,0],xmm6[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm6[2,3] -; SSE-NEXT: psrlq $48, %xmm6 -; SSE-NEXT: psrldq {{.*#+}} xmm7 = xmm7[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm9[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,5,7] -; SSE-NEXT: packuswb %xmm7, %xmm6 -; SSE-NEXT: movdqa %xmm5, %xmm7 -; SSE-NEXT: pandn %xmm6, %xmm7 -; SSE-NEXT: movdqa %xmm10, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm6[0,1,2,3,5,5,5,5] -; SSE-NEXT: movdqa {{.*#+}} xmm6 = [65535,0,65535,65535,0,65535,65535,65535] -; SSE-NEXT: movdqa %xmm6, %xmm11 -; SSE-NEXT: pandn %xmm9, %xmm11 -; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm2[0],xmm10[1],xmm2[1],xmm10[2],xmm2[2],xmm10[3],xmm2[3],xmm10[4],xmm2[4],xmm10[5],xmm2[5],xmm10[6],xmm2[6],xmm10[7],xmm2[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm10[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm9[3,1,1,2,4,5,6,7] -; SSE-NEXT: pand %xmm6, %xmm10 -; SSE-NEXT: por %xmm11, %xmm10 -; SSE-NEXT: packuswb %xmm10, %xmm10 -; SSE-NEXT: pand %xmm5, %xmm10 -; SSE-NEXT: por %xmm7, %xmm10 -; SSE-NEXT: movdqa %xmm12, %xmm7 -; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm2[8],xmm12[9],xmm2[9],xmm12[10],xmm2[10],xmm12[11],xmm2[11],xmm12[12],xmm2[12],xmm12[13],xmm2[13],xmm12[14],xmm2[14],xmm12[15],xmm2[15] -; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm12[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm9[0,1,2,3,5,5,7,4] -; SSE-NEXT: movdqa {{.*#+}} xmm9 = [65535,65535,65535,65535,0,65535,0,0] -; SSE-NEXT: movdqa %xmm9, %xmm12 -; SSE-NEXT: pandn %xmm11, %xmm12 -; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3],xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7] -; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,3,1,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,1,3,4,5,6,7] -; SSE-NEXT: pand %xmm9, %xmm7 -; SSE-NEXT: por %xmm7, %xmm12 -; SSE-NEXT: packuswb %xmm12, %xmm11 -; SSE-NEXT: movdqa %xmm0, %xmm7 -; SSE-NEXT: pandn %xmm11, %xmm7 -; SSE-NEXT: pand %xmm0, %xmm10 -; SSE-NEXT: por %xmm10, %xmm7 -; SSE-NEXT: movdqa %xmm14, %xmm10 -; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm2[8],xmm10[9],xmm2[9],xmm10[10],xmm2[10],xmm10[11],xmm2[11],xmm10[12],xmm2[12],xmm10[13],xmm2[13],xmm10[14],xmm2[14],xmm10[15],xmm2[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm2[0],xmm14[1],xmm2[1],xmm14[2],xmm2[2],xmm14[3],xmm2[3],xmm14[4],xmm2[4],xmm14[5],xmm2[5],xmm14[6],xmm2[6],xmm14[7],xmm2[7] -; SSE-NEXT: movdqa %xmm14, %xmm11 -; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[1,0],xmm10[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm14 = xmm14[2,0],xmm10[2,3] -; SSE-NEXT: psrlq $48, %xmm10 -; SSE-NEXT: psrldq {{.*#+}} xmm11 = xmm11[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm10[0],xmm11[1],xmm10[1],xmm11[2],xmm10[2],xmm11[3],xmm10[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm14[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,5,7] +; SSE-NEXT: pandn %xmm5, %xmm3 +; SSE-NEXT: pand %xmm0, %xmm6 +; SSE-NEXT: por %xmm6, %xmm3 +; SSE-NEXT: movdqa %xmm7, %xmm5 +; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3],xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7] +; SSE-NEXT: movdqa %xmm7, %xmm6 +; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[1,0],xmm5[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[2,0],xmm5[2,3] +; SSE-NEXT: psrlq $48, %xmm5 +; SSE-NEXT: psrldq {{.*#+}} xmm6 = xmm6[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm7[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm6, %xmm5 +; SSE-NEXT: movdqa %xmm2, %xmm6 +; SSE-NEXT: pandn %xmm5, %xmm6 +; SSE-NEXT: movdqa %xmm9, %xmm5 +; SSE-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm4[8],xmm5[9],xmm4[9],xmm5[10],xmm4[10],xmm5[11],xmm4[11],xmm5[12],xmm4[12],xmm5[13],xmm4[13],xmm5[14],xmm4[14],xmm5[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm5[0,1,2,3,5,5,5,5] +; SSE-NEXT: movdqa {{.*#+}} xmm5 = [65535,0,65535,65535,0,65535,65535,65535] +; SSE-NEXT: movdqa %xmm5, %xmm10 +; SSE-NEXT: pandn %xmm7, %xmm10 +; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3],xmm9[4],xmm4[4],xmm9[5],xmm4[5],xmm9[6],xmm4[6],xmm9[7],xmm4[7] +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm9[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm7[3,1,1,2,4,5,6,7] +; SSE-NEXT: pand %xmm5, %xmm9 +; SSE-NEXT: por %xmm10, %xmm9 +; SSE-NEXT: packuswb %xmm9, %xmm9 +; SSE-NEXT: pand %xmm2, %xmm9 +; SSE-NEXT: por %xmm6, %xmm9 +; SSE-NEXT: movdqa %xmm11, %xmm6 +; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm4[8],xmm11[9],xmm4[9],xmm11[10],xmm4[10],xmm11[11],xmm4[11],xmm11[12],xmm4[12],xmm11[13],xmm4[13],xmm11[14],xmm4[14],xmm11[15],xmm4[15] +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm11[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm7[0,1,2,3,5,5,7,4] +; SSE-NEXT: movdqa {{.*#+}} xmm7 = [65535,65535,65535,65535,0,65535,0,0] +; SSE-NEXT: movdqa %xmm7, %xmm11 +; SSE-NEXT: pandn %xmm10, %xmm11 +; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,3,1,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7] +; SSE-NEXT: pand %xmm7, %xmm6 +; SSE-NEXT: por %xmm6, %xmm11 ; SSE-NEXT: packuswb %xmm11, %xmm10 -; SSE-NEXT: movdqa %xmm1, %xmm11 -; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm2[8],xmm11[9],xmm2[9],xmm11[10],xmm2[10],xmm11[11],xmm2[11],xmm11[12],xmm2[12],xmm11[13],xmm2[13],xmm11[14],xmm2[14],xmm11[15],xmm2[15] -; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[1,1,2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,5,5,5,5] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3],xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm1[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm12[3,1,1,2,4,5,6,7] -; SSE-NEXT: pand %xmm6, %xmm12 -; SSE-NEXT: pandn %xmm11, %xmm6 -; SSE-NEXT: por %xmm12, %xmm6 -; SSE-NEXT: packuswb %xmm6, %xmm6 -; SSE-NEXT: pand %xmm5, %xmm6 +; SSE-NEXT: movdqa %xmm0, %xmm6 +; SSE-NEXT: pandn %xmm10, %xmm6 +; SSE-NEXT: pand %xmm0, %xmm9 +; SSE-NEXT: por %xmm9, %xmm6 +; SSE-NEXT: movdqa %xmm14, %xmm11 +; SSE-NEXT: movdqa %xmm14, %xmm9 +; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3],xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7] +; SSE-NEXT: movdqa %xmm11, %xmm10 +; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[1,0],xmm9[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm11 = xmm11[2,0],xmm9[2,3] +; SSE-NEXT: psrlq $48, %xmm9 +; SSE-NEXT: psrldq {{.*#+}} xmm10 = xmm10[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm11[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm10, %xmm9 +; SSE-NEXT: movdqa %xmm1, %xmm10 +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm4[8],xmm10[9],xmm4[9],xmm10[10],xmm4[10],xmm10[11],xmm4[11],xmm10[12],xmm4[12],xmm10[13],xmm4[13],xmm10[14],xmm4[14],xmm10[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5] +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] +; SSE-NEXT: pshufhw {{.*#+}} xmm11 = xmm1[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm11 = xmm11[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm11 = xmm11[3,1,1,2,4,5,6,7] +; SSE-NEXT: pand %xmm5, %xmm11 ; SSE-NEXT: pandn %xmm10, %xmm5 -; SSE-NEXT: por %xmm5, %xmm6 -; SSE-NEXT: movdqa %xmm13, %xmm5 -; SSE-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm2[8],xmm13[9],xmm2[9],xmm13[10],xmm2[10],xmm13[11],xmm2[11],xmm13[12],xmm2[12],xmm13[13],xmm2[13],xmm13[14],xmm2[14],xmm13[15],xmm2[15] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,3,1,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] -; SSE-NEXT: pand %xmm9, %xmm4 -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm13[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,7,4] -; SSE-NEXT: pandn %xmm5, %xmm9 -; SSE-NEXT: por %xmm4, %xmm9 -; SSE-NEXT: pand %xmm0, %xmm6 -; SSE-NEXT: packuswb %xmm9, %xmm4 -; SSE-NEXT: pandn %xmm4, %xmm0 -; SSE-NEXT: por %xmm6, %xmm0 -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: movaps %xmm4, 16(%rsi) -; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: movaps %xmm4, (%rsi) +; SSE-NEXT: por %xmm11, %xmm5 +; SSE-NEXT: packuswb %xmm5, %xmm5 +; SSE-NEXT: pand %xmm2, %xmm5 +; SSE-NEXT: pandn %xmm9, %xmm2 +; SSE-NEXT: por %xmm2, %xmm5 +; SSE-NEXT: movdqa %xmm12, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3],xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm4[8],xmm12[9],xmm4[9],xmm12[10],xmm4[10],xmm12[11],xmm4[11],xmm12[12],xmm4[12],xmm12[13],xmm4[13],xmm12[14],xmm4[14],xmm12[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,1,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7] +; SSE-NEXT: pand %xmm7, %xmm2 +; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm12[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,5,7,4] +; SSE-NEXT: pandn %xmm4, %xmm7 +; SSE-NEXT: por %xmm2, %xmm7 +; SSE-NEXT: pand %xmm0, %xmm5 +; SSE-NEXT: packuswb %xmm7, %xmm2 +; SSE-NEXT: pandn %xmm2, %xmm0 +; SSE-NEXT: por %xmm5, %xmm0 +; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: movaps %xmm2, 16(%rsi) +; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: movaps %xmm2, (%rsi) ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; SSE-NEXT: movaps %xmm1, 16(%rdx) ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload @@ -1698,15 +1687,15 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: movaps %xmm1, 16(%rcx) ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; SSE-NEXT: movaps %xmm1, (%rcx) +; SSE-NEXT: movdqa %xmm13, 16(%r8) ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: movaps %xmm1, 16(%r8) -; SSE-NEXT: movdqa %xmm15, (%r8) +; SSE-NEXT: movaps %xmm1, (%r8) ; SSE-NEXT: movdqa %xmm3, 16(%r9) ; SSE-NEXT: movdqa %xmm8, (%r9) ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax ; SSE-NEXT: movdqa %xmm0, 16(%rax) -; SSE-NEXT: movdqa %xmm7, (%rax) -; SSE-NEXT: addq $280, %rsp # imm = 0x118 +; SSE-NEXT: movdqa %xmm6, (%rax) +; SSE-NEXT: addq $264, %rsp # imm = 0x108 ; SSE-NEXT: retq ; ; AVX1-ONLY-LABEL: load_i8_stride6_vf32: @@ -2299,7 +2288,7 @@ define void @load_i8_stride6_vf32(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr %out.vec2, ptr %out.vec3, ptr %out.vec4, ptr %out.vec5) nounwind { ; SSE-LABEL: load_i8_stride6_vf64: ; SSE: # %bb.0: -; SSE-NEXT: subq $808, %rsp # imm = 0x328 +; SSE-NEXT: subq $792, %rsp # imm = 0x318 ; SSE-NEXT: movdqa 64(%rdi), %xmm4 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 80(%rdi), %xmm5 @@ -2311,22 +2300,22 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: movdqa 32(%rdi), %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 48(%rdi), %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm9 = [65535,65535,0,65535,65535,0,65535,65535] -; SSE-NEXT: movdqa %xmm9, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm13 = [65535,65535,0,65535,65535,0,65535,65535] +; SSE-NEXT: movdqa %xmm13, %xmm1 ; SSE-NEXT: pandn %xmm2, %xmm1 ; SSE-NEXT: movdqa {{.*#+}} xmm3 = [65535,0,65535,65535,0,65535,65535,0] ; SSE-NEXT: movdqa %xmm3, %xmm2 ; SSE-NEXT: pandn %xmm0, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm9, %xmm2 +; SSE-NEXT: movdqa %xmm13, %xmm2 ; SSE-NEXT: pandn %xmm0, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm9, %xmm0 +; SSE-NEXT: pand %xmm13, %xmm0 ; SSE-NEXT: por %xmm1, %xmm0 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] ; SSE-NEXT: pand %xmm1, %xmm0 -; SSE-NEXT: movdqa %xmm1, %xmm11 +; SSE-NEXT: movdqa %xmm1, %xmm10 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] @@ -2339,9 +2328,8 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pand %xmm3, %xmm2 ; SSE-NEXT: por %xmm1, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm2, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,1,3] +; SSE-NEXT: pand %xmm10, %xmm1 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,3,2,1,4,5,6,7] @@ -2351,15 +2339,13 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: movdqa %xmm8, %xmm2 ; SSE-NEXT: pandn %xmm0, %xmm2 ; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm9, %xmm6 -; SSE-NEXT: movdqa %xmm9, %xmm0 +; SSE-NEXT: movdqa %xmm13, %xmm0 ; SSE-NEXT: pandn %xmm5, %xmm0 -; SSE-NEXT: pand %xmm9, %xmm4 +; SSE-NEXT: pand %xmm13, %xmm4 ; SSE-NEXT: por %xmm0, %xmm4 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm4[3,1,2,0] +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5] @@ -2372,282 +2358,272 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 320(%rdi), %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm9, %xmm0 +; SSE-NEXT: movdqa %xmm13, %xmm0 ; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: movdqa 336(%rdi), %xmm13 +; SSE-NEXT: movdqa 336(%rdi), %xmm12 ; SSE-NEXT: movdqa %xmm3, %xmm1 -; SSE-NEXT: pandn %xmm13, %xmm1 +; SSE-NEXT: pandn %xmm12, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm9, %xmm1 -; SSE-NEXT: pandn %xmm13, %xmm1 +; SSE-NEXT: movdqa %xmm13, %xmm1 +; SSE-NEXT: pandn %xmm12, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm9, %xmm13 -; SSE-NEXT: por %xmm0, %xmm13 -; SSE-NEXT: movdqa %xmm13, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: pand %xmm13, %xmm12 +; SSE-NEXT: por %xmm0, %xmm12 +; SSE-NEXT: movdqa %xmm12, %xmm0 +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] ; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa %xmm8, %xmm2 -; SSE-NEXT: pandn %xmm0, %xmm2 -; SSE-NEXT: movdqa 304(%rdi), %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm8, %xmm1 +; SSE-NEXT: pandn %xmm0, %xmm1 +; SSE-NEXT: movdqa 304(%rdi), %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm3, %xmm7 ; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: movdqa 288(%rdi), %xmm7 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: pand %xmm3, %xmm1 -; SSE-NEXT: por %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm1, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSE-NEXT: pandn %xmm2, %xmm0 +; SSE-NEXT: movdqa 288(%rdi), %xmm6 +; SSE-NEXT: movdqa %xmm6, %xmm2 +; SSE-NEXT: pand %xmm3, %xmm2 +; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,2,1,3] +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: packuswb %xmm0, %xmm0 ; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: por %xmm2, %xmm0 +; SSE-NEXT: por %xmm1, %xmm0 ; SSE-NEXT: movdqa 368(%rdi), %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm9, %xmm2 +; SSE-NEXT: movdqa %xmm13, %xmm2 ; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: movdqa 352(%rdi), %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm9, %xmm1 -; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm1, %xmm2 -; SSE-NEXT: pand %xmm11, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,1,2,0] +; SSE-NEXT: movdqa 352(%rdi), %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm13, %xmm3 +; SSE-NEXT: por %xmm2, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[3,1,2,0] +; SSE-NEXT: pand %xmm10, %xmm2 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5] ; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm4, %xmm1 -; SSE-NEXT: pandn %xmm2, %xmm1 +; SSE-NEXT: movdqa %xmm4, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm3 ; SSE-NEXT: pand %xmm4, %xmm0 ; SSE-NEXT: movdqa %xmm4, %xmm9 -; SSE-NEXT: por %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: por %xmm0, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 224(%rdi), %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm0 +; SSE-NEXT: movdqa %xmm13, %xmm0 ; SSE-NEXT: pandn %xmm1, %xmm0 -; SSE-NEXT: movdqa 240(%rdi), %xmm14 -; SSE-NEXT: movdqa %xmm3, %xmm1 -; SSE-NEXT: pandn %xmm14, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm1 -; SSE-NEXT: pandn %xmm14, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm6, %xmm14 -; SSE-NEXT: por %xmm0, %xmm14 -; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: movdqa 240(%rdi), %xmm11 +; SSE-NEXT: movdqa %xmm7, %xmm2 +; SSE-NEXT: pandn %xmm11, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm13, %xmm2 +; SSE-NEXT: pandn %xmm11, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm13, %xmm11 +; SSE-NEXT: por %xmm0, %xmm11 +; SSE-NEXT: movdqa %xmm11, %xmm0 +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] ; SSE-NEXT: packuswb %xmm2, %xmm0 -; SSE-NEXT: movdqa %xmm8, %xmm5 ; SSE-NEXT: movdqa %xmm8, %xmm2 ; SSE-NEXT: pandn %xmm0, %xmm2 -; SSE-NEXT: movdqa 208(%rdi), %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm3, %xmm1 -; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: pandn %xmm4, %xmm0 +; SSE-NEXT: movdqa 208(%rdi), %xmm1 +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: pandn %xmm1, %xmm0 ; SSE-NEXT: movdqa 192(%rdi), %xmm3 -; SSE-NEXT: movdqa %xmm3, %xmm4 -; SSE-NEXT: pand %xmm1, %xmm4 -; SSE-NEXT: movdqa %xmm1, %xmm8 -; SSE-NEXT: por %xmm0, %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] +; SSE-NEXT: movdqa %xmm3, %xmm1 +; SSE-NEXT: pand %xmm7, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,1,3] +; SSE-NEXT: movdqa %xmm10, %xmm1 +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pand %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm5, %xmm1 +; SSE-NEXT: pand %xmm8, %xmm0 +; SSE-NEXT: movdqa %xmm8, %xmm10 ; SSE-NEXT: por %xmm2, %xmm0 -; SSE-NEXT: movdqa 272(%rdi), %xmm15 -; SSE-NEXT: movdqa %xmm6, %xmm2 -; SSE-NEXT: pandn %xmm15, %xmm2 -; SSE-NEXT: movdqa 256(%rdi), %xmm12 -; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm6, %xmm12 -; SSE-NEXT: por %xmm2, %xmm12 -; SSE-NEXT: movdqa %xmm12, %xmm2 -; SSE-NEXT: pand %xmm11, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,1,2,0] +; SSE-NEXT: movdqa 272(%rdi), %xmm14 +; SSE-NEXT: movdqa %xmm13, %xmm2 +; SSE-NEXT: pandn %xmm14, %xmm2 +; SSE-NEXT: movdqa 256(%rdi), %xmm15 +; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm13, %xmm15 +; SSE-NEXT: por %xmm2, %xmm15 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm15[3,1,2,0] +; SSE-NEXT: pand %xmm1, %xmm2 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,5] ; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm9, %xmm4 ; SSE-NEXT: pandn %xmm2, %xmm4 ; SSE-NEXT: pand %xmm9, %xmm0 ; SSE-NEXT: por %xmm0, %xmm4 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa 128(%rdi), %xmm2 ; SSE-NEXT: movdqa %xmm2, (%rsp) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm0 +; SSE-NEXT: movdqa %xmm13, %xmm0 ; SSE-NEXT: pandn %xmm2, %xmm0 -; SSE-NEXT: movdqa 144(%rdi), %xmm10 -; SSE-NEXT: movdqa %xmm8, %xmm4 -; SSE-NEXT: pandn %xmm10, %xmm4 +; SSE-NEXT: movdqa 144(%rdi), %xmm9 +; SSE-NEXT: movdqa %xmm7, %xmm4 +; SSE-NEXT: pandn %xmm9, %xmm4 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm4 -; SSE-NEXT: pandn %xmm10, %xmm4 +; SSE-NEXT: movdqa %xmm13, %xmm4 +; SSE-NEXT: pandn %xmm9, %xmm4 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm6, %xmm10 -; SSE-NEXT: por %xmm0, %xmm10 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: pand %xmm13, %xmm9 +; SSE-NEXT: por %xmm0, %xmm9 +; SSE-NEXT: movdqa %xmm9, %xmm0 +; SSE-NEXT: pand %xmm1, %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,3,2,3] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,6,5] ; SSE-NEXT: packuswb %xmm5, %xmm0 -; SSE-NEXT: movdqa %xmm1, %xmm11 -; SSE-NEXT: pandn %xmm0, %xmm11 -; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: movdqa %xmm6, %xmm4 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: pandn %xmm1, %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm2 -; SSE-NEXT: pandn %xmm7, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm10 +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: movdqa %xmm13, %xmm2 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload +; SSE-NEXT: pandn %xmm13, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm4 -; SSE-NEXT: movdqa %xmm6, %xmm2 +; SSE-NEXT: movdqa %xmm13, %xmm1 +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: pandn %xmm6, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm6, %xmm5 +; SSE-NEXT: movdqa %xmm0, %xmm2 ; SSE-NEXT: pandn %xmm3, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm3, %xmm5 +; SSE-NEXT: movdqa %xmm3, %xmm4 ; SSE-NEXT: movdqa 112(%rdi), %xmm6 -; SSE-NEXT: movdqa %xmm8, %xmm7 -; SSE-NEXT: movdqa %xmm8, %xmm9 -; SSE-NEXT: pandn %xmm6, %xmm9 -; SSE-NEXT: movdqa 160(%rdi), %xmm8 -; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm8 +; SSE-NEXT: movdqa %xmm7, %xmm2 +; SSE-NEXT: movdqa %xmm7, %xmm8 +; SSE-NEXT: pandn %xmm6, %xmm8 +; SSE-NEXT: movdqa 160(%rdi), %xmm7 +; SSE-NEXT: movdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm7 ; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload -; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload +; SSE-NEXT: pandn %xmm13, %xmm3 ; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm1 +; SSE-NEXT: movdqa %xmm2, %xmm3 +; SSE-NEXT: movdqa %xmm2, %xmm1 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm0, %xmm2 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: pandn %xmm3, %xmm1 +; SSE-NEXT: pandn %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm5 +; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm1, %xmm5 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: pandn %xmm1, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, %xmm3 +; SSE-NEXT: movdqa %xmm0, %xmm2 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: pandn %xmm1, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pandn %xmm1, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm0, %xmm4 ; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: pandn %xmm4, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm4, %xmm3 -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload -; SSE-NEXT: pandn %xmm4, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm5 +; SSE-NEXT: pandn %xmm14, %xmm5 ; SSE-NEXT: movdqa %xmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm3 -; SSE-NEXT: pandn %xmm15, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm15 -; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, %xmm3 -; SSE-NEXT: pandn %xmm6, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm6, %xmm15 -; SSE-NEXT: movdqa 96(%rdi), %xmm5 -; SSE-NEXT: movdqa %xmm5, %xmm3 -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa 176(%rdi), %xmm6 -; SSE-NEXT: movdqa %xmm6, %xmm3 -; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm3 -; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm14 +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: pandn %xmm6, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 96(%rdi), %xmm4 +; SSE-NEXT: movdqa %xmm4, %xmm2 +; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa 176(%rdi), %xmm14 +; SSE-NEXT: movdqa %xmm14, %xmm2 +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm0, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm0, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm13 +; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm0, %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pand %xmm0, %xmm1 ; SSE-NEXT: movdqa %xmm1, (%rsp) # 16-byte Spill ; SSE-NEXT: movdqa %xmm0, %xmm1 -; SSE-NEXT: pand %xmm0, %xmm15 -; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm0, %xmm6 +; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm0, %xmm13 ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pandn %xmm5, %xmm1 +; SSE-NEXT: pandn %xmm4, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm5, %xmm3 -; SSE-NEXT: pand %xmm7, %xmm3 -; SSE-NEXT: por %xmm9, %xmm3 -; SSE-NEXT: movdqa %xmm3, %xmm5 -; SSE-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm7, %xmm5 -; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,1,3] +; SSE-NEXT: movdqa %xmm4, %xmm3 +; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm3 +; SSE-NEXT: por %xmm8, %xmm3 +; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,2,1,3] +; SSE-NEXT: movdqa {{.*#+}} xmm1 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm1, %xmm5 ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,6,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm5 = xmm5[0,3,2,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,4,7,6,7] ; SSE-NEXT: packuswb %xmm5, %xmm5 -; SSE-NEXT: movdqa {{.*#+}} xmm9 = [65535,65535,65535,0,0,0,65535,65535] -; SSE-NEXT: pand %xmm9, %xmm5 -; SSE-NEXT: por %xmm11, %xmm5 -; SSE-NEXT: pandn %xmm6, %xmm0 -; SSE-NEXT: por %xmm0, %xmm8 -; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: pand %xmm7, %xmm0 -; SSE-NEXT: movdqa %xmm7, %xmm11 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] +; SSE-NEXT: movdqa {{.*#+}} xmm8 = [65535,65535,65535,0,0,0,65535,65535] +; SSE-NEXT: pand %xmm8, %xmm5 +; SSE-NEXT: por %xmm10, %xmm5 +; SSE-NEXT: pandn %xmm14, %xmm0 +; SSE-NEXT: por %xmm0, %xmm7 +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm7[3,1,2,0] +; SSE-NEXT: pand %xmm1, %xmm0 ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,5] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm7 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] -; SSE-NEXT: movdqa %xmm7, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] +; SSE-NEXT: movdqa %xmm10, %xmm1 ; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm7, %xmm5 +; SSE-NEXT: pand %xmm10, %xmm5 ; SSE-NEXT: por %xmm5, %xmm1 ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: pxor %xmm5, %xmm5 @@ -2658,11 +2634,11 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,2,3,3] ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3] ; SSE-NEXT: psrld $16, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm1[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,5,7,6,7] -; SSE-NEXT: punpckhdq {{.*#+}} xmm15 = xmm15[2],xmm0[2],xmm15[3],xmm0[3] -; SSE-NEXT: packuswb %xmm15, %xmm4 -; SSE-NEXT: movdqa %xmm9, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm1[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,5,7,6,7] +; SSE-NEXT: punpckhdq {{.*#+}} xmm14 = xmm14[2],xmm0[2],xmm14[3],xmm0[3] +; SSE-NEXT: packuswb %xmm14, %xmm4 +; SSE-NEXT: movdqa %xmm8, %xmm1 ; SSE-NEXT: pandn %xmm4, %xmm1 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload ; SSE-NEXT: movdqa %xmm2, %xmm4 @@ -2671,48 +2647,48 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,1,1,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,7,6,7] ; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,65535,0,65535,0,0,65535,65535] -; SSE-NEXT: movdqa %xmm0, %xmm15 -; SSE-NEXT: pandn %xmm4, %xmm15 +; SSE-NEXT: movdqa %xmm0, %xmm14 +; SSE-NEXT: pandn %xmm4, %xmm14 ; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm2[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,3,2,0,4,5,6,7] ; SSE-NEXT: pand %xmm0, %xmm4 -; SSE-NEXT: por %xmm15, %xmm4 +; SSE-NEXT: por %xmm14, %xmm4 ; SSE-NEXT: packuswb %xmm4, %xmm4 -; SSE-NEXT: pand %xmm9, %xmm4 +; SSE-NEXT: pand %xmm8, %xmm4 ; SSE-NEXT: por %xmm1, %xmm4 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload ; SSE-NEXT: movdqa %xmm6, %xmm1 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm1[0,1,2,3,5,5,5,5] +; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm1[0,1,2,3,5,5,5,5] ; SSE-NEXT: movdqa {{.*#+}} xmm2 = [65535,65535,65535,0,65535,65535,0,65535] ; SSE-NEXT: movdqa %xmm2, %xmm1 -; SSE-NEXT: pandn %xmm15, %xmm1 +; SSE-NEXT: pandn %xmm14, %xmm1 ; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] -; SSE-NEXT: pshuflw {{.*#+}} xmm15 = xmm6[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm15 = xmm15[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm15 = xmm15[0,1,2,3,5,7,6,4] -; SSE-NEXT: pand %xmm2, %xmm15 -; SSE-NEXT: por %xmm1, %xmm15 -; SSE-NEXT: packuswb %xmm15, %xmm1 -; SSE-NEXT: movdqa %xmm7, %xmm15 -; SSE-NEXT: pandn %xmm1, %xmm15 -; SSE-NEXT: pand %xmm7, %xmm4 -; SSE-NEXT: por %xmm4, %xmm15 -; SSE-NEXT: movdqa %xmm15, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm13, %xmm1 +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm6[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm14[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm14[0,1,2,3,5,7,6,4] +; SSE-NEXT: pand %xmm2, %xmm14 +; SSE-NEXT: por %xmm1, %xmm14 +; SSE-NEXT: packuswb %xmm14, %xmm1 +; SSE-NEXT: movdqa %xmm10, %xmm14 +; SSE-NEXT: pandn %xmm1, %xmm14 +; SSE-NEXT: pand %xmm10, %xmm4 +; SSE-NEXT: por %xmm4, %xmm14 +; SSE-NEXT: movdqa %xmm14, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm12, %xmm1 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm13 = xmm13[0],xmm5[0],xmm13[1],xmm5[1],xmm13[2],xmm5[2],xmm13[3],xmm5[3],xmm13[4],xmm5[4],xmm13[5],xmm5[5],xmm13[6],xmm5[6],xmm13[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm13[2,2,3,3] +; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3],xmm12[4],xmm5[4],xmm12[5],xmm5[5],xmm12[6],xmm5[6],xmm12[7],xmm5[7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm12[2,2,3,3] ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3] ; SSE-NEXT: psrld $16, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm13 = xmm13[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm13 = xmm13[0,1,2,3,5,7,6,7] -; SSE-NEXT: punpckhdq {{.*#+}} xmm13 = xmm13[2],xmm1[2],xmm13[3],xmm1[3] -; SSE-NEXT: packuswb %xmm13, %xmm4 -; SSE-NEXT: movdqa %xmm9, %xmm15 -; SSE-NEXT: movdqa %xmm9, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,5,7,6,7] +; SSE-NEXT: punpckhdq {{.*#+}} xmm12 = xmm12[2],xmm1[2],xmm12[3],xmm1[3] +; SSE-NEXT: packuswb %xmm12, %xmm4 +; SSE-NEXT: movdqa %xmm8, %xmm14 +; SSE-NEXT: movdqa %xmm8, %xmm1 ; SSE-NEXT: pandn %xmm4, %xmm1 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload ; SSE-NEXT: movdqa %xmm6, %xmm4 @@ -2720,46 +2696,46 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,1,0,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,1,1,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,7,6,7] -; SSE-NEXT: movdqa %xmm0, %xmm13 -; SSE-NEXT: pandn %xmm4, %xmm13 +; SSE-NEXT: movdqa %xmm0, %xmm12 +; SSE-NEXT: pandn %xmm4, %xmm12 ; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm6[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,3,2,0,4,5,6,7] ; SSE-NEXT: pand %xmm0, %xmm4 -; SSE-NEXT: por %xmm13, %xmm4 +; SSE-NEXT: por %xmm12, %xmm4 ; SSE-NEXT: packuswb %xmm4, %xmm4 -; SSE-NEXT: pand %xmm9, %xmm4 +; SSE-NEXT: pand %xmm8, %xmm4 ; SSE-NEXT: por %xmm1, %xmm4 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload ; SSE-NEXT: movdqa %xmm6, %xmm1 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: movdqa %xmm2, %xmm13 -; SSE-NEXT: pandn %xmm1, %xmm13 +; SSE-NEXT: movdqa %xmm2, %xmm12 +; SSE-NEXT: pandn %xmm1, %xmm12 ; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm6[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,7,6,4] ; SSE-NEXT: pand %xmm2, %xmm1 -; SSE-NEXT: por %xmm13, %xmm1 +; SSE-NEXT: por %xmm12, %xmm1 ; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm7, %xmm13 -; SSE-NEXT: pandn %xmm1, %xmm13 -; SSE-NEXT: pand %xmm7, %xmm4 -; SSE-NEXT: por %xmm4, %xmm13 -; SSE-NEXT: movdqa %xmm13, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm14, %xmm1 +; SSE-NEXT: movdqa %xmm10, %xmm12 +; SSE-NEXT: pandn %xmm1, %xmm12 +; SSE-NEXT: pand %xmm10, %xmm4 +; SSE-NEXT: por %xmm4, %xmm12 +; SSE-NEXT: movdqa %xmm12, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm11, %xmm1 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3],xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm14[2,2,3,3] +; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1],xmm11[2],xmm5[2],xmm11[3],xmm5[3],xmm11[4],xmm5[4],xmm11[5],xmm5[5],xmm11[6],xmm5[6],xmm11[7],xmm5[7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm11[2,2,3,3] ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3] ; SSE-NEXT: psrld $16, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm14[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,7,6,7] -; SSE-NEXT: punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm1[2],xmm9[3],xmm1[3] -; SSE-NEXT: packuswb %xmm9, %xmm4 -; SSE-NEXT: movdqa %xmm15, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm11[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,7,6,7] +; SSE-NEXT: punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm1[2],xmm8[3],xmm1[3] +; SSE-NEXT: packuswb %xmm8, %xmm4 +; SSE-NEXT: movdqa %xmm14, %xmm1 ; SSE-NEXT: pandn %xmm4, %xmm1 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload ; SSE-NEXT: movdqa %xmm6, %xmm4 @@ -2767,41 +2743,41 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[2,1,0,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,1,1,1,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm4[0,1,2,3,5,7,6,7] -; SSE-NEXT: movdqa %xmm0, %xmm9 -; SSE-NEXT: pandn %xmm4, %xmm9 +; SSE-NEXT: movdqa %xmm0, %xmm8 +; SSE-NEXT: pandn %xmm4, %xmm8 ; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm6[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[1,3,2,0,4,5,6,7] ; SSE-NEXT: pand %xmm0, %xmm4 -; SSE-NEXT: por %xmm9, %xmm4 +; SSE-NEXT: por %xmm8, %xmm4 ; SSE-NEXT: packuswb %xmm4, %xmm4 -; SSE-NEXT: pand %xmm15, %xmm4 +; SSE-NEXT: pand %xmm14, %xmm4 ; SSE-NEXT: por %xmm1, %xmm4 -; SSE-NEXT: movdqa %xmm12, %xmm1 +; SSE-NEXT: movdqa %xmm15, %xmm1 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: movdqa %xmm2, %xmm9 -; SSE-NEXT: pandn %xmm1, %xmm9 -; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm5[8],xmm12[9],xmm5[9],xmm12[10],xmm5[10],xmm12[11],xmm5[11],xmm12[12],xmm5[12],xmm12[13],xmm5[13],xmm12[14],xmm5[14],xmm12[15],xmm5[15] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm12[3,1,2,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm2, %xmm8 +; SSE-NEXT: pandn %xmm1, %xmm8 +; SSE-NEXT: punpckhbw {{.*#+}} xmm15 = xmm15[8],xmm5[8],xmm15[9],xmm5[9],xmm15[10],xmm5[10],xmm15[11],xmm5[11],xmm15[12],xmm5[12],xmm15[13],xmm5[13],xmm15[14],xmm5[14],xmm15[15],xmm5[15] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm15[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,7,6,4] ; SSE-NEXT: pand %xmm2, %xmm1 -; SSE-NEXT: por %xmm9, %xmm1 +; SSE-NEXT: por %xmm8, %xmm1 ; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm7, %xmm9 -; SSE-NEXT: pandn %xmm1, %xmm9 -; SSE-NEXT: pand %xmm7, %xmm4 -; SSE-NEXT: por %xmm4, %xmm9 -; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm10, %xmm1 +; SSE-NEXT: movdqa %xmm10, %xmm8 +; SSE-NEXT: pandn %xmm1, %xmm8 +; SSE-NEXT: pand %xmm10, %xmm4 +; SSE-NEXT: por %xmm4, %xmm8 +; SSE-NEXT: movdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm9, %xmm1 ; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3],xmm10[4],xmm5[4],xmm10[5],xmm5[5],xmm10[6],xmm5[6],xmm10[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm10[2,2,3,3] +; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3],xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm9[2,2,3,3] ; SSE-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3] ; SSE-NEXT: psrld $16, %xmm1 -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm10[0,1,0,3] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm9[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,7,6,7] ; SSE-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm1[2],xmm6[3],xmm1[3] ; SSE-NEXT: packuswb %xmm6, %xmm4 @@ -2818,32 +2794,34 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: pandn %xmm1, %xmm0 ; SSE-NEXT: por %xmm3, %xmm0 ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa %xmm15, %xmm1 -; SSE-NEXT: pand %xmm15, %xmm0 +; SSE-NEXT: movdqa %xmm14, %xmm1 +; SSE-NEXT: pand %xmm14, %xmm0 ; SSE-NEXT: pandn %xmm4, %xmm1 ; SSE-NEXT: por %xmm1, %xmm0 -; SSE-NEXT: movdqa %xmm8, %xmm1 +; SSE-NEXT: movdqa %xmm7, %xmm1 ; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15] -; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm8[3,1,2,3,4,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm5[8],xmm7[9],xmm5[9],xmm7[10],xmm5[10],xmm7[11],xmm5[11],xmm7[12],xmm5[12],xmm7[13],xmm5[13],xmm7[14],xmm5[14],xmm7[15],xmm5[15] +; SSE-NEXT: pshuflw {{.*#+}} xmm3 = xmm7[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,5,7,6,4] ; SSE-NEXT: pand %xmm2, %xmm3 ; SSE-NEXT: pandn %xmm1, %xmm2 ; SSE-NEXT: por %xmm3, %xmm2 ; SSE-NEXT: packuswb %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm7, %xmm2 +; SSE-NEXT: movdqa %xmm10, %xmm2 ; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm7, %xmm0 +; SSE-NEXT: pand %xmm10, %xmm0 +; SSE-NEXT: movdqa %xmm10, %xmm11 ; SSE-NEXT: por %xmm0, %xmm2 ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{.*#+}} xmm12 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; SSE-NEXT: pand %xmm12, %xmm0 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm11, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm14 = [65535,0,65535,65535,0,65535,65535,0] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload +; SSE-NEXT: pand %xmm14, %xmm4 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm4, %xmm0 +; SSE-NEXT: movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,3,4,5,6,7] @@ -2853,719 +2831,712 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: packuswb %xmm1, %xmm0 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm6, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: movdqa %xmm11, %xmm4 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm6[2,1,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm10, %xmm1 +; SSE-NEXT: movdqa %xmm10, %xmm12 ; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa {{.*#+}} xmm3 = [0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255] -; SSE-NEXT: movdqa %xmm3, %xmm2 -; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm3, %xmm0 -; SSE-NEXT: por %xmm0, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,5,5,5,5] +; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: movdqa {{.*#+}} xmm5 = [0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255] +; SSE-NEXT: movdqa %xmm5, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: por %xmm0, %xmm3 +; SSE-NEXT: movdqa %xmm13, %xmm0 ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload -; SSE-NEXT: por %xmm0, %xmm14 -; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm1[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm7, %xmm2 -; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: pand %xmm12, %xmm1 -; SSE-NEXT: movdqa %xmm12, %xmm10 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm1, %xmm0 -; SSE-NEXT: movdqa %xmm1, %xmm12 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm11, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pand %xmm11, %xmm3 +; SSE-NEXT: movdqa %xmm11, %xmm8 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload +; SSE-NEXT: pand %xmm14, %xmm13 +; SSE-NEXT: movdqa %xmm14, %xmm7 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: pand %xmm10, %xmm0 +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,3,4,5,6,7] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,5,6] -; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm15, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm3, %xmm2 -; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm3, %xmm0 -; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm14[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm10, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5] +; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: movdqa %xmm5, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: por %xmm0, %xmm3 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; SSE-NEXT: por %xmm0, %xmm9 -; SSE-NEXT: movdqa %xmm9, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload +; SSE-NEXT: por %xmm0, %xmm11 +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm11[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm10, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm7, %xmm2 -; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload -; SSE-NEXT: pand %xmm10, %xmm8 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm8, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pand %xmm8, %xmm3 +; SSE-NEXT: movdqa %xmm8, %xmm9 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload +; SSE-NEXT: pand %xmm7, %xmm10 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: pand %xmm12, %xmm0 +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,3,4,5,6,7] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,5,6] -; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm13, %xmm1 -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm3, %xmm2 -; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm3, %xmm0 -; SSE-NEXT: por %xmm0, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE-NEXT: packuswb %xmm2, %xmm0 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: por %xmm0, %xmm1 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload ; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm1, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm1[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm12, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5] +; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: movdqa %xmm5, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: por %xmm0, %xmm3 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload +; SSE-NEXT: por %xmm0, %xmm8 +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm8[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm12, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa %xmm7, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm7, %xmm2 -; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload -; SSE-NEXT: pand %xmm10, %xmm7 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm7, %xmm0 -; SSE-NEXT: pand %xmm11, %xmm0 -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,3,4,5,6,7] +; SSE-NEXT: movdqa %xmm9, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pand %xmm9, %xmm3 +; SSE-NEXT: movdqa %xmm9, %xmm1 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE-NEXT: pand %xmm7, %xmm0 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pand %xmm12, %xmm0 +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm0[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,3,4,5,6,7] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,3] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,4,5,6] -; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm11, %xmm1 -; SSE-NEXT: pand %xmm4, %xmm1 -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,5,4,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[1,2,3,0,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,5,5,5,5] -; SSE-NEXT: packuswb %xmm1, %xmm1 -; SSE-NEXT: movdqa %xmm3, %xmm2 -; SSE-NEXT: pandn %xmm1, %xmm2 -; SSE-NEXT: pand %xmm3, %xmm0 -; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: packuswb %xmm2, %xmm0 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm15[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,4,7] +; SSE-NEXT: pand %xmm12, %xmm2 +; SSE-NEXT: movdqa %xmm12, %xmm9 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[1,2,3,0,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5] +; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: movdqa %xmm5, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: pand %xmm5, %xmm0 +; SSE-NEXT: por %xmm0, %xmm3 ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; SSE-NEXT: por %xmm0, %xmm10 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: pand %xmm4, %xmm0 -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,3,2,3,4,5,6,7] +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload +; SSE-NEXT: por %xmm0, %xmm12 +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm12[0,3,2,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] +; SSE-NEXT: pand %xmm9, %xmm0 ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,2,2,2,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,5,6,7,4] ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] -; SSE-NEXT: movdqa %xmm4, %xmm1 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: pand %xmm4, %xmm2 -; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload +; SSE-NEXT: movdqa %xmm1, %xmm9 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pand %xmm1, %xmm3 +; SSE-NEXT: por %xmm3, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa %xmm4, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15] -; SSE-NEXT: movdqa %xmm4, %xmm1 -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm0[3,0] -; SSE-NEXT: movaps %xmm0, %xmm2 -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,1],xmm1[0,2] +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15] +; SSE-NEXT: pxor %xmm7, %xmm7 +; SSE-NEXT: movdqa %xmm4, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[3,0] +; SSE-NEXT: movaps %xmm0, %xmm3 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,1],xmm2[0,2] ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm4[0,0] ; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm4[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm2[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm0, %xmm1 +; SSE-NEXT: packuswb %xmm0, %xmm2 ; SSE-NEXT: movdqa %xmm6, %xmm0 -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1],xmm0[2],xmm7[2],xmm0[3],xmm7[3],xmm0[4],xmm7[4],xmm0[5],xmm7[5],xmm0[6],xmm7[6],xmm0[7],xmm7[7] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm0[3,1,2,1,4,5,6,7] -; SSE-NEXT: movdqa {{.*#+}} xmm0 = [0,65535,65535,0,65535,65535,65535,65535] -; SSE-NEXT: movdqa %xmm0, %xmm4 -; SSE-NEXT: pandn %xmm2, %xmm4 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm6[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7] -; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: por %xmm4, %xmm2 -; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7] +; SSE-NEXT: movdqa {{.*#+}} xmm3 = [0,65535,65535,0,65535,65535,65535,65535] ; SSE-NEXT: movdqa %xmm3, %xmm4 -; SSE-NEXT: pandn %xmm2, %xmm4 -; SSE-NEXT: pand %xmm3, %xmm1 -; SSE-NEXT: por %xmm1, %xmm4 -; SSE-NEXT: movdqa %xmm14, %xmm1 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,2,1] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm1[0,1,2,3,7,5,6,5] -; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535,0,65535,65535,0] -; SSE-NEXT: movdqa %xmm1, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3],xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm14[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7] -; SSE-NEXT: pand %xmm1, %xmm2 -; SSE-NEXT: por %xmm6, %xmm2 -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa {{.*#+}} xmm14 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0] -; SSE-NEXT: movdqa %xmm14, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: pand %xmm14, %xmm4 -; SSE-NEXT: por %xmm4, %xmm6 -; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm12, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm12 = xmm12[8],xmm5[8],xmm12[9],xmm5[9],xmm12[10],xmm5[10],xmm12[11],xmm5[11],xmm12[12],xmm5[12],xmm12[13],xmm5[13],xmm12[14],xmm5[14],xmm12[15],xmm5[15] -; SSE-NEXT: movdqa %xmm12, %xmm4 -; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0] -; SSE-NEXT: movaps %xmm2, %xmm6 -; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm4[0,2] -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm12[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm12[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm6[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,2] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm4 -; SSE-NEXT: movdqa %xmm15, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,1,4,5,6,7] -; SSE-NEXT: movdqa %xmm0, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm15 = xmm15[8],xmm5[8],xmm15[9],xmm5[9],xmm15[10],xmm5[10],xmm15[11],xmm5[11],xmm15[12],xmm5[12],xmm15[13],xmm5[13],xmm15[14],xmm5[14],xmm15[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm15[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7] -; SSE-NEXT: pand %xmm0, %xmm2 -; SSE-NEXT: por %xmm6, %xmm2 -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm3, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: pand %xmm3, %xmm4 -; SSE-NEXT: por %xmm4, %xmm6 +; SSE-NEXT: pandn %xmm0, %xmm4 +; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm7[8],xmm6[9],xmm7[9],xmm6[10],xmm7[10],xmm6[11],xmm7[11],xmm6[12],xmm7[12],xmm6[13],xmm7[13],xmm6[14],xmm7[14],xmm6[15],xmm7[15] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm6[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7] +; SSE-NEXT: pand %xmm3, %xmm0 +; SSE-NEXT: por %xmm4, %xmm0 +; SSE-NEXT: packuswb %xmm0, %xmm0 +; SSE-NEXT: movdqa %xmm5, %xmm6 +; SSE-NEXT: pandn %xmm0, %xmm6 +; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: por %xmm2, %xmm6 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload +; SSE-NEXT: movdqa %xmm1, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm7[8],xmm0[9],xmm7[9],xmm0[10],xmm7[10],xmm0[11],xmm7[11],xmm0[12],xmm7[12],xmm0[13],xmm7[13],xmm0[14],xmm7[14],xmm0[15],xmm7[15] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,5] +; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,65535,0,65535,65535,0] +; SSE-NEXT: movdqa %xmm4, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm7[0],xmm1[1],xmm7[1],xmm1[2],xmm7[2],xmm1[3],xmm7[3],xmm1[4],xmm7[4],xmm1[5],xmm7[5],xmm1[6],xmm7[6],xmm1[7],xmm7[7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7] +; SSE-NEXT: pand %xmm4, %xmm0 +; SSE-NEXT: por %xmm2, %xmm0 +; SSE-NEXT: packuswb %xmm0, %xmm0 ; SSE-NEXT: movdqa %xmm9, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,5] -; SSE-NEXT: movdqa %xmm1, %xmm4 -; SSE-NEXT: pandn %xmm2, %xmm4 -; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3],xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm9[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7] -; SSE-NEXT: pand %xmm1, %xmm2 -; SSE-NEXT: por %xmm4, %xmm2 -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm14, %xmm4 -; SSE-NEXT: pandn %xmm2, %xmm4 -; SSE-NEXT: pand %xmm14, %xmm6 -; SSE-NEXT: por %xmm6, %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm8, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15] -; SSE-NEXT: movdqa %xmm8, %xmm4 -; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0] -; SSE-NEXT: movaps %xmm2, %xmm6 -; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm4[0,2] -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm8[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm8[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm6[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,2] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm4 -; SSE-NEXT: movdqa %xmm13, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,1,4,5,6,7] -; SSE-NEXT: movdqa %xmm0, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm5[8],xmm13[9],xmm5[9],xmm13[10],xmm5[10],xmm13[11],xmm5[11],xmm13[12],xmm5[12],xmm13[13],xmm5[13],xmm13[14],xmm5[14],xmm13[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm13[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,7,7,7] -; SSE-NEXT: pand %xmm0, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pand %xmm9, %xmm6 ; SSE-NEXT: por %xmm6, %xmm2 -; SSE-NEXT: packuswb %xmm2, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm13, %xmm0 +; SSE-NEXT: pxor %xmm1, %xmm1 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm1[8],xmm13[9],xmm1[9],xmm13[10],xmm1[10],xmm13[11],xmm1[11],xmm13[12],xmm1[12],xmm13[13],xmm1[13],xmm13[14],xmm1[14],xmm13[15],xmm1[15] +; SSE-NEXT: movdqa %xmm13, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[3,0] +; SSE-NEXT: movaps %xmm0, %xmm6 +; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm2[0,2] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm13[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm13[2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm6[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm14, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7] ; SSE-NEXT: movdqa %xmm3, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: pand %xmm3, %xmm4 -; SSE-NEXT: por %xmm4, %xmm6 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm8 # 16-byte Reload -; SSE-NEXT: movdqa %xmm8, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,5] -; SSE-NEXT: movdqa %xmm1, %xmm4 -; SSE-NEXT: pandn %xmm2, %xmm4 -; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3],xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm8[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7] -; SSE-NEXT: pand %xmm1, %xmm2 -; SSE-NEXT: por %xmm4, %xmm2 -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm14, %xmm4 -; SSE-NEXT: pandn %xmm2, %xmm4 -; SSE-NEXT: pand %xmm14, %xmm6 -; SSE-NEXT: por %xmm6, %xmm4 -; SSE-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm7, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm5[8],xmm7[9],xmm5[9],xmm7[10],xmm5[10],xmm7[11],xmm5[11],xmm7[12],xmm5[12],xmm7[13],xmm5[13],xmm7[14],xmm5[14],xmm7[15],xmm5[15] -; SSE-NEXT: movdqa %xmm7, %xmm4 -; SSE-NEXT: shufps {{.*#+}} xmm4 = xmm4[2,0],xmm2[3,0] -; SSE-NEXT: movaps %xmm2, %xmm6 -; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm4[0,2] -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm7[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm7[2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm4 = xmm6[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,2] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm2, %xmm4 -; SSE-NEXT: movdqa %xmm11, %xmm2 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,1,4,5,6,7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm5[8],xmm11[9],xmm5[9],xmm11[10],xmm5[10],xmm11[11],xmm5[11],xmm11[12],xmm5[12],xmm11[13],xmm5[13],xmm11[14],xmm5[14],xmm11[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm11[0,3,2,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,3,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,7,7,7,7] -; SSE-NEXT: pand %xmm0, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm0 +; SSE-NEXT: pandn %xmm0, %xmm6 +; SSE-NEXT: punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm1[8],xmm14[9],xmm1[9],xmm14[10],xmm1[10],xmm14[11],xmm1[11],xmm14[12],xmm1[12],xmm14[13],xmm1[13],xmm14[14],xmm1[14],xmm14[15],xmm1[15] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm14[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7] +; SSE-NEXT: pand %xmm3, %xmm0 ; SSE-NEXT: por %xmm6, %xmm0 -; SSE-NEXT: pand %xmm3, %xmm4 ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pandn %xmm0, %xmm3 -; SSE-NEXT: por %xmm4, %xmm3 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm5[8],xmm0[9],xmm5[9],xmm0[10],xmm5[10],xmm0[11],xmm5[11],xmm0[12],xmm5[12],xmm0[13],xmm5[13],xmm0[14],xmm5[14],xmm0[15],xmm5[15] +; SSE-NEXT: movdqa %xmm5, %xmm6 +; SSE-NEXT: pandn %xmm0, %xmm6 +; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: por %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm11, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] ; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] ; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,5] -; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3],xmm10[4],xmm5[4],xmm10[5],xmm5[5],xmm10[6],xmm5[6],xmm10[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm10[0,2,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,5,7,7] -; SSE-NEXT: pand %xmm1, %xmm2 -; SSE-NEXT: pandn %xmm0, %xmm1 -; SSE-NEXT: por %xmm2, %xmm1 -; SSE-NEXT: movdqa %xmm14, %xmm2 -; SSE-NEXT: pand %xmm14, %xmm3 -; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: pandn %xmm0, %xmm2 -; SSE-NEXT: por %xmm3, %xmm2 -; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm8 = [255,255,255,255,255,255,255,255] -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2] -; SSE-NEXT: packuswb %xmm1, %xmm0 -; SSE-NEXT: movdqa {{.*#+}} xmm13 = [255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255] -; SSE-NEXT: movdqa %xmm13, %xmm2 +; SSE-NEXT: movdqa %xmm4, %xmm2 ; SSE-NEXT: pandn %xmm0, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm11, %xmm0 -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pand %xmm13, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm1[0],xmm11[1],xmm1[1],xmm11[2],xmm1[2],xmm11[3],xmm1[3],xmm11[4],xmm1[4],xmm11[5],xmm1[5],xmm11[6],xmm1[6],xmm11[7],xmm1[7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm11[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7] +; SSE-NEXT: pand %xmm4, %xmm0 ; SSE-NEXT: por %xmm2, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; SSE-NEXT: movdqa {{.*#+}} xmm6 = [65535,0,65535,65535,0,65535,65535,0] -; SSE-NEXT: pand %xmm6, %xmm9 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload +; SSE-NEXT: packuswb %xmm0, %xmm0 ; SSE-NEXT: movdqa %xmm9, %xmm2 -; SSE-NEXT: pand %xmm8, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,1,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa {{.*#+}} xmm4 = [65535,65535,65535,65535,65535,0,0,0] -; SSE-NEXT: movdqa %xmm4, %xmm1 -; SSE-NEXT: pandn %xmm2, %xmm1 -; SSE-NEXT: pand %xmm4, %xmm0 -; SSE-NEXT: por %xmm0, %xmm1 -; SSE-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm12, %xmm0 -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2] -; SSE-NEXT: packuswb %xmm2, %xmm0 -; SSE-NEXT: movdqa %xmm13, %xmm2 ; SSE-NEXT: pandn %xmm0, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm3, %xmm0 -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7] +; SSE-NEXT: pand %xmm9, %xmm6 +; SSE-NEXT: por %xmm6, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa %xmm10, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm1[8],xmm10[9],xmm1[9],xmm10[10],xmm1[10],xmm10[11],xmm1[11],xmm10[12],xmm1[12],xmm10[13],xmm1[13],xmm10[14],xmm1[14],xmm10[15],xmm1[15] +; SSE-NEXT: movdqa %xmm10, %xmm2 +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[3,0] +; SSE-NEXT: movaps %xmm0, %xmm6 +; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[0,1],xmm2[0,2] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm10[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm10[2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm6[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm0, %xmm2 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload +; SSE-NEXT: movdqa %xmm7, %xmm0 +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[3,1,2,1,4,5,6,7] +; SSE-NEXT: movdqa %xmm3, %xmm6 +; SSE-NEXT: pandn %xmm0, %xmm6 +; SSE-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8],xmm1[8],xmm7[9],xmm1[9],xmm7[10],xmm1[10],xmm7[11],xmm1[11],xmm7[12],xmm1[12],xmm7[13],xmm1[13],xmm7[14],xmm1[14],xmm7[15],xmm1[15] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm7[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,7,7,7] +; SSE-NEXT: pand %xmm3, %xmm0 +; SSE-NEXT: por %xmm6, %xmm0 ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pand %xmm13, %xmm0 -; SSE-NEXT: por %xmm2, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload -; SSE-NEXT: pand %xmm6, %xmm1 -; SSE-NEXT: movdqa %xmm6, %xmm7 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm1, %xmm2 -; SSE-NEXT: pand %xmm8, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,1,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm4, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm5, %xmm6 +; SSE-NEXT: pandn %xmm0, %xmm6 +; SSE-NEXT: pand %xmm5, %xmm2 +; SSE-NEXT: por %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm8, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,2,1] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,7,5,6,5] +; SSE-NEXT: movdqa %xmm4, %xmm2 +; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm1[0],xmm8[1],xmm1[1],xmm8[2],xmm1[2],xmm8[3],xmm1[3],xmm8[4],xmm1[4],xmm8[5],xmm1[5],xmm8[6],xmm1[6],xmm8[7],xmm1[7] +; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm8[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5,7,7] ; SSE-NEXT: pand %xmm4, %xmm0 -; SSE-NEXT: por %xmm0, %xmm6 -; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: por %xmm2, %xmm0 +; SSE-NEXT: packuswb %xmm0, %xmm2 +; SSE-NEXT: movdqa %xmm9, %xmm0 +; SSE-NEXT: pandn %xmm2, %xmm0 +; SSE-NEXT: pand %xmm9, %xmm6 +; SSE-NEXT: por %xmm6, %xmm0 +; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload +; SSE-NEXT: movdqa %xmm0, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15] +; SSE-NEXT: movdqa %xmm0, %xmm6 +; SSE-NEXT: shufps {{.*#+}} xmm6 = xmm6[2,0],xmm2[3,0] +; SSE-NEXT: movaps %xmm2, %xmm7 +; SSE-NEXT: shufps {{.*#+}} xmm7 = xmm7[0,1],xmm6[0,2] +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,0],xmm0[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,0],xmm0[2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm7[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,2] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,3,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm15, %xmm2 +; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[3,1,2,1,4,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm15 = xmm15[8],xmm1[8],xmm15[9],xmm1[9],xmm15[10],xmm1[10],xmm15[11],xmm1[11],xmm15[12],xmm1[12],xmm15[13],xmm1[13],xmm15[14],xmm1[14],xmm15[15],xmm1[15] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm15[0,3,2,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,3,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,7,7,7,7] +; SSE-NEXT: pand %xmm3, %xmm7 +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: por %xmm7, %xmm3 +; SSE-NEXT: pand %xmm5, %xmm6 +; SSE-NEXT: packuswb %xmm3, %xmm3 +; SSE-NEXT: pandn %xmm3, %xmm5 +; SSE-NEXT: por %xmm6, %xmm5 +; SSE-NEXT: movdqa %xmm12, %xmm2 +; SSE-NEXT: pxor %xmm0, %xmm0 +; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm0[8],xmm2[9],xmm0[9],xmm2[10],xmm0[10],xmm2[11],xmm0[11],xmm2[12],xmm0[12],xmm2[13],xmm0[13],xmm2[14],xmm0[14],xmm2[15],xmm0[15] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,2,1] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,7,5,6,5] +; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm0[0],xmm12[1],xmm0[1],xmm12[2],xmm0[2],xmm12[3],xmm0[3],xmm12[4],xmm0[4],xmm12[5],xmm0[5],xmm12[6],xmm0[6],xmm12[7],xmm0[7] +; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm12[0,2,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm3 = xmm3[0,1,2,3,4,5,7,7] +; SSE-NEXT: pand %xmm4, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm4 +; SSE-NEXT: por %xmm3, %xmm4 +; SSE-NEXT: pand %xmm9, %xmm5 +; SSE-NEXT: packuswb %xmm4, %xmm2 +; SSE-NEXT: pandn %xmm2, %xmm9 +; SSE-NEXT: por %xmm5, %xmm9 +; SSE-NEXT: movdqa %xmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,1,2,3] +; SSE-NEXT: movdqa %xmm0, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm10 = [255,255,255,255,255,255,255,255] +; SSE-NEXT: pand %xmm10, %xmm1 +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,1,2,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2] -; SSE-NEXT: packuswb %xmm2, %xmm0 -; SSE-NEXT: movdqa %xmm13, %xmm2 -; SSE-NEXT: pandn %xmm0, %xmm2 +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,1,0,2] +; SSE-NEXT: packuswb %xmm2, %xmm1 +; SSE-NEXT: movdqa {{.*#+}} xmm15 = [255,255,255,255,255,0,0,0,0,0,255,255,255,255,255,255] +; SSE-NEXT: movdqa %xmm15, %xmm2 +; SSE-NEXT: pandn %xmm1, %xmm2 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm12[3,1,2,0] +; SSE-NEXT: pand %xmm10, %xmm1 +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,1,0,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm1[2,1,0,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm6, %xmm6 +; SSE-NEXT: pand %xmm15, %xmm6 +; SSE-NEXT: por %xmm2, %xmm6 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Reload +; SSE-NEXT: movdqa {{.*#+}} xmm11 = [65535,0,65535,65535,0,65535,65535,0] +; SSE-NEXT: pand %xmm11, %xmm13 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm13 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm13[0,2,1,3] +; SSE-NEXT: pand %xmm10, %xmm1 +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,2,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,1,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,0,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm1 = xmm1[0,1,2,3,6,5,4,7] +; SSE-NEXT: packuswb %xmm1, %xmm2 +; SSE-NEXT: movdqa {{.*#+}} xmm1 = [65535,65535,65535,65535,65535,0,0,0] +; SSE-NEXT: movdqa %xmm1, %xmm3 +; SSE-NEXT: pandn %xmm2, %xmm3 +; SSE-NEXT: pand %xmm1, %xmm6 +; SSE-NEXT: por %xmm6, %xmm3 +; SSE-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill ; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload ; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm14, %xmm0 -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[2,1,0,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pand %xmm13, %xmm0 -; SSE-NEXT: por %xmm2, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Reload -; SSE-NEXT: pand %xmm7, %xmm15 -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm15, %xmm2 -; SSE-NEXT: pand %xmm8, %xmm2 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] +; SSE-NEXT: movdqa %xmm14, %xmm2 +; SSE-NEXT: pand %xmm10, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,2] +; SSE-NEXT: packuswb %xmm6, %xmm2 +; SSE-NEXT: movdqa %xmm15, %xmm6 +; SSE-NEXT: pandn %xmm2, %xmm6 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm4[3,1,2,0] +; SSE-NEXT: pand %xmm10, %xmm2 +; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,1,0,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm2[2,1,0,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm7, %xmm7 +; SSE-NEXT: pand %xmm15, %xmm7 +; SSE-NEXT: por %xmm6, %xmm7 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: pand %xmm11, %xmm3 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm3[0,2,1,3] +; SSE-NEXT: pand %xmm10, %xmm2 ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,2,1,4,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,1,3] ; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,0,3,4,5,6,7] ; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm2, %xmm2 -; SSE-NEXT: movdqa %xmm4, %xmm6 -; SSE-NEXT: pandn %xmm2, %xmm6 -; SSE-NEXT: pand %xmm4, %xmm0 -; SSE-NEXT: por %xmm0, %xmm6 +; SSE-NEXT: packuswb %xmm2, %xmm6 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: pandn %xmm6, %xmm2 +; SSE-NEXT: pand %xmm1, %xmm7 +; SSE-NEXT: por %xmm7, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload ; SSE-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa (%rsp), %xmm0 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm0, (%rsp) # 16-byte Spill -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,1,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[2,1,2,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,7,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,1,0,2] -; SSE-NEXT: packuswb %xmm2, %xmm0 -; SSE-NEXT: movdqa %xmm13, %xmm2 -; SSE-NEXT: pandn %xmm0, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload -; SSE-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: pand %xmm8, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,1,0,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm0[2,1,0,3,4,5,6,7] -; SSE-NEXT: packuswb %xmm6, %xmm6 -; SSE-NEXT: pand %xmm13, %xmm6 -; SSE-NEXT: por %xmm2, %xmm6 -; SSE-NEXT: movdqa %xmm7, %xmm2 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload -; SSE-NEXT: pand %xmm7, %xmm0 -; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload -; SSE-NEXT: por %xmm0, %xmm2 +; SSE-NEXT: pand %xmm10, %xmm6 +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm6[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,2] +; SSE-NEXT: packuswb %xmm7, %xmm6 +; SSE-NEXT: movdqa %xmm15, %xmm7 +; SSE-NEXT: pandn %xmm6, %xmm7 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload ; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill -; SSE-NEXT: movdqa %xmm8, %xmm0 -; SSE-NEXT: pand %xmm2, %xmm0 -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,2,1,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,1,0,3,4,5,6,7] -; SSE-NEXT: pshufhw {{.*#+}} xmm0 = xmm0[0,1,2,3,6,5,4,7] -; SSE-NEXT: packuswb %xmm0, %xmm2 -; SSE-NEXT: movdqa %xmm4, %xmm7 -; SSE-NEXT: pandn %xmm2, %xmm7 -; SSE-NEXT: pand %xmm4, %xmm6 -; SSE-NEXT: por %xmm6, %xmm7 -; SSE-NEXT: movdqa %xmm10, %xmm0 -; SSE-NEXT: movdqa %xmm10, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] -; SSE-NEXT: movdqa %xmm0, %xmm6 -; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm2[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm2[2,3] -; SSE-NEXT: psrlq $48, %xmm2 -; SSE-NEXT: psrldq {{.*#+}} xmm6 = xmm6[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm0[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,5,7] -; SSE-NEXT: packuswb %xmm6, %xmm2 -; SSE-NEXT: movdqa %xmm13, %xmm8 -; SSE-NEXT: pandn %xmm2, %xmm8 +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[3,1,2,0] +; SSE-NEXT: pand %xmm10, %xmm6 +; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,6,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[2,1,0,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm6[2,1,0,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm8, %xmm8 +; SSE-NEXT: pand %xmm15, %xmm8 +; SSE-NEXT: por %xmm7, %xmm8 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Reload +; SSE-NEXT: pand %xmm11, %xmm2 +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm2[0,2,1,3] +; SSE-NEXT: pand %xmm10, %xmm6 +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,2,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,1,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,0,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,6,5,4,7] +; SSE-NEXT: packuswb %xmm6, %xmm7 +; SSE-NEXT: movdqa %xmm1, %xmm2 +; SSE-NEXT: pandn %xmm7, %xmm2 +; SSE-NEXT: pand %xmm1, %xmm8 +; SSE-NEXT: por %xmm8, %xmm2 +; SSE-NEXT: movdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; SSE-NEXT: movdqa (%rsp), %xmm7 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Folded Reload +; SSE-NEXT: movdqa %xmm7, (%rsp) # 16-byte Spill +; SSE-NEXT: pand %xmm10, %xmm7 +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm7[2,1,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[2,1,2,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,4,7,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,1,0,2] +; SSE-NEXT: packuswb %xmm8, %xmm7 +; SSE-NEXT: movdqa %xmm15, %xmm8 +; SSE-NEXT: pandn %xmm7, %xmm8 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload +; SSE-NEXT: por {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Folded Reload +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm6[3,1,2,0] +; SSE-NEXT: pand %xmm10, %xmm7 +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[2,1,0,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm7[2,1,0,3,4,5,6,7] +; SSE-NEXT: packuswb %xmm9, %xmm9 +; SSE-NEXT: pand %xmm15, %xmm9 +; SSE-NEXT: por %xmm8, %xmm9 ; SSE-NEXT: movdqa %xmm11, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,5,5,5,5] -; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,0,65535,65535,0,65535,65535,65535] -; SSE-NEXT: movdqa %xmm0, %xmm10 -; SSE-NEXT: pandn %xmm2, %xmm10 -; SSE-NEXT: punpcklbw {{.*#+}} xmm11 = xmm11[0],xmm5[0],xmm11[1],xmm5[1],xmm11[2],xmm5[2],xmm11[3],xmm5[3],xmm11[4],xmm5[4],xmm11[5],xmm5[5],xmm11[6],xmm5[6],xmm11[7],xmm5[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm2 = xmm11[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm2[3,1,1,2,4,5,6,7] -; SSE-NEXT: pand %xmm0, %xmm6 -; SSE-NEXT: por %xmm10, %xmm6 -; SSE-NEXT: packuswb %xmm6, %xmm6 -; SSE-NEXT: pand %xmm13, %xmm6 -; SSE-NEXT: por %xmm8, %xmm6 -; SSE-NEXT: movdqa %xmm9, %xmm2 -; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm5[8],xmm9[9],xmm5[9],xmm9[10],xmm5[10],xmm9[11],xmm5[11],xmm9[12],xmm5[12],xmm9[13],xmm5[13],xmm9[14],xmm5[14],xmm9[15],xmm5[15] -; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm9[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,5,7,4] -; SSE-NEXT: movdqa {{.*#+}} xmm11 = [65535,65535,65535,65535,0,65535,0,0] -; SSE-NEXT: movdqa %xmm11, %xmm10 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload +; SSE-NEXT: pand %xmm11, %xmm7 +; SSE-NEXT: pandn {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload +; SSE-NEXT: por %xmm7, %xmm2 +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm2[0,2,1,3] +; SSE-NEXT: pand %xmm10, %xmm7 +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,2,1,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,2,1,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm7 = xmm7[0,1,0,3,4,5,6,7] +; SSE-NEXT: pshufhw {{.*#+}} xmm7 = xmm7[0,1,2,3,6,5,4,7] +; SSE-NEXT: packuswb %xmm7, %xmm8 +; SSE-NEXT: movdqa %xmm1, %xmm7 +; SSE-NEXT: pandn %xmm8, %xmm7 +; SSE-NEXT: pand %xmm1, %xmm9 +; SSE-NEXT: por %xmm9, %xmm7 +; SSE-NEXT: movdqa %xmm0, %xmm8 +; SSE-NEXT: pxor %xmm5, %xmm5 +; SSE-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1],xmm0[2],xmm5[2],xmm0[3],xmm5[3],xmm0[4],xmm5[4],xmm0[5],xmm5[5],xmm0[6],xmm5[6],xmm0[7],xmm5[7] +; SSE-NEXT: movdqa %xmm0, %xmm9 +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[1,0],xmm8[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm0 = xmm0[2,0],xmm8[2,3] +; SSE-NEXT: psrlq $48, %xmm8 +; SSE-NEXT: psrldq {{.*#+}} xmm9 = xmm9[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm0[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm9, %xmm8 +; SSE-NEXT: movdqa %xmm15, %xmm10 ; SSE-NEXT: pandn %xmm8, %xmm10 -; SSE-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3],xmm2[4],xmm5[4],xmm2[5],xmm5[5],xmm2[6],xmm5[6],xmm2[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,3,1,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,1,1,3,4,5,6,7] -; SSE-NEXT: pand %xmm11, %xmm2 -; SSE-NEXT: por %xmm2, %xmm10 -; SSE-NEXT: packuswb %xmm10, %xmm8 -; SSE-NEXT: movdqa %xmm4, %xmm2 -; SSE-NEXT: pandn %xmm8, %xmm2 -; SSE-NEXT: pand %xmm4, %xmm6 -; SSE-NEXT: por %xmm6, %xmm2 -; SSE-NEXT: movdqa %xmm12, %xmm9 -; SSE-NEXT: movdqa %xmm12, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3],xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7] -; SSE-NEXT: movdqa %xmm9, %xmm8 -; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,0],xmm6[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm6[2,3] -; SSE-NEXT: psrlq $48, %xmm6 -; SSE-NEXT: psrldq {{.*#+}} xmm8 = xmm8[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm9[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,5,7] -; SSE-NEXT: packuswb %xmm8, %xmm6 +; SSE-NEXT: movdqa %xmm12, %xmm8 +; SSE-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15] +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[1,1,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,5,5,5] +; SSE-NEXT: movdqa {{.*#+}} xmm0 = [65535,0,65535,65535,0,65535,65535,65535] +; SSE-NEXT: movdqa %xmm0, %xmm11 +; SSE-NEXT: pandn %xmm8, %xmm11 +; SSE-NEXT: punpcklbw {{.*#+}} xmm12 = xmm12[0],xmm5[0],xmm12[1],xmm5[1],xmm12[2],xmm5[2],xmm12[3],xmm5[3],xmm12[4],xmm5[4],xmm12[5],xmm5[5],xmm12[6],xmm5[6],xmm12[7],xmm5[7] +; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm12[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm12 = xmm8[3,1,1,2,4,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm12 +; SSE-NEXT: por %xmm11, %xmm12 +; SSE-NEXT: packuswb %xmm12, %xmm12 +; SSE-NEXT: pand %xmm15, %xmm12 +; SSE-NEXT: por %xmm10, %xmm12 ; SSE-NEXT: movdqa %xmm13, %xmm8 -; SSE-NEXT: pandn %xmm6, %xmm8 -; SSE-NEXT: movdqa %xmm3, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,5,5,5] -; SSE-NEXT: movdqa %xmm0, %xmm10 -; SSE-NEXT: pandn %xmm6, %xmm10 -; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm3[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[3,1,1,2,4,5,6,7] -; SSE-NEXT: pand %xmm0, %xmm6 -; SSE-NEXT: por %xmm10, %xmm6 -; SSE-NEXT: packuswb %xmm6, %xmm6 -; SSE-NEXT: pand %xmm13, %xmm6 -; SSE-NEXT: por %xmm8, %xmm6 -; SSE-NEXT: movdqa %xmm1, %xmm8 -; SSE-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm5[8],xmm1[9],xmm5[9],xmm1[10],xmm5[10],xmm1[11],xmm5[11],xmm1[12],xmm5[12],xmm1[13],xmm5[13],xmm1[14],xmm5[14],xmm1[15],xmm5[15] -; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm1[0,1,2,3,7,5,6,7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm13 = xmm13[8],xmm5[8],xmm13[9],xmm5[9],xmm13[10],xmm5[10],xmm13[11],xmm5[11],xmm13[12],xmm5[12],xmm13[13],xmm5[13],xmm13[14],xmm5[14],xmm13[15],xmm5[15] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm13[0,1,2,3,7,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,7,4] -; SSE-NEXT: movdqa %xmm11, %xmm12 -; SSE-NEXT: pandn %xmm10, %xmm12 +; SSE-NEXT: movdqa {{.*#+}} xmm11 = [65535,65535,65535,65535,0,65535,0,0] +; SSE-NEXT: movdqa %xmm11, %xmm13 +; SSE-NEXT: pandn %xmm10, %xmm13 ; SSE-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3],xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7] ; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,3,1,1] ; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm8[0,1,1,3,4,5,6,7] ; SSE-NEXT: pand %xmm11, %xmm8 -; SSE-NEXT: por %xmm8, %xmm12 -; SSE-NEXT: packuswb %xmm12, %xmm8 -; SSE-NEXT: movdqa %xmm4, %xmm12 -; SSE-NEXT: pandn %xmm8, %xmm12 -; SSE-NEXT: pand %xmm4, %xmm6 -; SSE-NEXT: por %xmm6, %xmm12 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload -; SSE-NEXT: movdqa %xmm9, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3],xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7] -; SSE-NEXT: movdqa %xmm9, %xmm8 -; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,0],xmm6[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm6[2,3] -; SSE-NEXT: psrlq $48, %xmm6 -; SSE-NEXT: psrldq {{.*#+}} xmm8 = xmm8[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm6[0],xmm8[1],xmm6[1],xmm8[2],xmm6[2],xmm8[3],xmm6[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm9[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,4,4,5,7] -; SSE-NEXT: packuswb %xmm8, %xmm6 -; SSE-NEXT: movdqa %xmm13, %xmm8 -; SSE-NEXT: pandn %xmm6, %xmm8 -; SSE-NEXT: movdqa %xmm14, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm5[8],xmm6[9],xmm5[9],xmm6[10],xmm5[10],xmm6[11],xmm5[11],xmm6[12],xmm5[12],xmm6[13],xmm5[13],xmm6[14],xmm5[14],xmm6[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm6[0,1,2,3,5,5,5,5] -; SSE-NEXT: movdqa %xmm0, %xmm9 -; SSE-NEXT: pandn %xmm6, %xmm9 -; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3],xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm6 = xmm14[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm6[3,1,1,2,4,5,6,7] -; SSE-NEXT: pand %xmm0, %xmm10 -; SSE-NEXT: por %xmm9, %xmm10 -; SSE-NEXT: packuswb %xmm10, %xmm10 -; SSE-NEXT: pand %xmm13, %xmm10 -; SSE-NEXT: por %xmm8, %xmm10 -; SSE-NEXT: movdqa %xmm15, %xmm6 -; SSE-NEXT: punpckhbw {{.*#+}} xmm15 = xmm15[8],xmm5[8],xmm15[9],xmm5[9],xmm15[10],xmm5[10],xmm15[11],xmm5[11],xmm15[12],xmm5[12],xmm15[13],xmm5[13],xmm15[14],xmm5[14],xmm15[15],xmm5[15] -; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm15[0,1,2,3,7,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,2,0] -; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,5,5,7,4] -; SSE-NEXT: movdqa %xmm11, %xmm9 -; SSE-NEXT: pandn %xmm8, %xmm9 -; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] -; SSE-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,3,1,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm6 = xmm6[0,1,1,3,4,5,6,7] -; SSE-NEXT: pand %xmm11, %xmm6 -; SSE-NEXT: por %xmm6, %xmm9 -; SSE-NEXT: packuswb %xmm9, %xmm8 -; SSE-NEXT: movdqa %xmm4, %xmm6 -; SSE-NEXT: pandn %xmm8, %xmm6 -; SSE-NEXT: pand %xmm4, %xmm10 -; SSE-NEXT: por %xmm10, %xmm6 -; SSE-NEXT: movdqa (%rsp), %xmm1 # 16-byte Reload +; SSE-NEXT: por %xmm8, %xmm13 +; SSE-NEXT: packuswb %xmm13, %xmm10 ; SSE-NEXT: movdqa %xmm1, %xmm8 -; SSE-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8],xmm5[8],xmm8[9],xmm5[9],xmm8[10],xmm5[10],xmm8[11],xmm5[11],xmm8[12],xmm5[12],xmm8[13],xmm5[13],xmm8[14],xmm5[14],xmm8[15],xmm5[15] -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] -; SSE-NEXT: movdqa %xmm1, %xmm9 -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,0],xmm8[0,0] -; SSE-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0],xmm8[2,3] -; SSE-NEXT: psrlq $48, %xmm8 -; SSE-NEXT: psrldq {{.*#+}} xmm9 = xmm9[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; SSE-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm8[0],xmm9[1],xmm8[1],xmm9[2],xmm8[2],xmm9[3],xmm8[3] -; SSE-NEXT: pshuflw {{.*#+}} xmm8 = xmm1[3,1,2,3,4,5,6,7] -; SSE-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,1,0,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,4,5,7] -; SSE-NEXT: packuswb %xmm9, %xmm8 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload +; SSE-NEXT: pandn %xmm10, %xmm8 +; SSE-NEXT: pand %xmm1, %xmm12 +; SSE-NEXT: por %xmm12, %xmm8 ; SSE-NEXT: movdqa %xmm14, %xmm9 -; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm5[8],xmm9[9],xmm5[9],xmm9[10],xmm5[10],xmm9[11],xmm5[11],xmm9[12],xmm5[12],xmm9[13],xmm5[13],xmm9[14],xmm5[14],xmm9[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[1,1,2,3] -; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,5,5,5,5] -; SSE-NEXT: punpcklbw {{.*#+}} xmm14 = xmm14[0],xmm5[0],xmm14[1],xmm5[1],xmm14[2],xmm5[2],xmm14[3],xmm5[3],xmm14[4],xmm5[4],xmm14[5],xmm5[5],xmm14[6],xmm5[6],xmm14[7],xmm5[7] -; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm14[0,1,2,3,7,5,6,7] +; SSE-NEXT: movdqa %xmm14, %xmm10 +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm5[8],xmm10[9],xmm5[9],xmm10[10],xmm5[10],xmm10[11],xmm5[11],xmm10[12],xmm5[12],xmm10[13],xmm5[13],xmm10[14],xmm5[14],xmm10[15],xmm5[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1],xmm9[2],xmm5[2],xmm9[3],xmm5[3],xmm9[4],xmm5[4],xmm9[5],xmm5[5],xmm9[6],xmm5[6],xmm9[7],xmm5[7] +; SSE-NEXT: movdqa %xmm9, %xmm12 +; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,0],xmm10[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm10[2,3] +; SSE-NEXT: psrlq $48, %xmm10 +; SSE-NEXT: psrldq {{.*#+}} xmm12 = xmm12[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm12 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm9[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm12, %xmm10 +; SSE-NEXT: movdqa %xmm15, %xmm12 +; SSE-NEXT: pandn %xmm10, %xmm12 +; SSE-NEXT: movdqa %xmm4, %xmm10 +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm5[8],xmm10[9],xmm5[9],xmm10[10],xmm5[10],xmm10[11],xmm5[11],xmm10[12],xmm5[12],xmm10[13],xmm5[13],xmm10[14],xmm5[14],xmm10[15],xmm5[15] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5] +; SSE-NEXT: movdqa %xmm0, %xmm14 +; SSE-NEXT: pandn %xmm10, %xmm14 +; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm4[0,1,2,3,7,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,2,2,3] -; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[3,1,1,2,4,5,6,7] -; SSE-NEXT: pand %xmm0, %xmm10 -; SSE-NEXT: pandn %xmm9, %xmm0 -; SSE-NEXT: por %xmm10, %xmm0 +; SSE-NEXT: pshuflw {{.*#+}} xmm13 = xmm10[3,1,1,2,4,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm13 +; SSE-NEXT: por %xmm14, %xmm13 +; SSE-NEXT: packuswb %xmm13, %xmm13 +; SSE-NEXT: pand %xmm15, %xmm13 +; SSE-NEXT: por %xmm12, %xmm13 +; SSE-NEXT: movdqa %xmm3, %xmm10 +; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm5[8],xmm3[9],xmm5[9],xmm3[10],xmm5[10],xmm3[11],xmm5[11],xmm3[12],xmm5[12],xmm3[13],xmm5[13],xmm3[14],xmm5[14],xmm3[15],xmm5[15] +; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm12 = xmm12[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm12 = xmm12[0,1,2,3,5,5,7,4] +; SSE-NEXT: movdqa %xmm11, %xmm14 +; SSE-NEXT: pandn %xmm12, %xmm14 +; SSE-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm5[0],xmm10[1],xmm5[1],xmm10[2],xmm5[2],xmm10[3],xmm5[3],xmm10[4],xmm5[4],xmm10[5],xmm5[5],xmm10[6],xmm5[6],xmm10[7],xmm5[7] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,3,1,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm10[0,1,1,3,4,5,6,7] +; SSE-NEXT: pand %xmm11, %xmm10 +; SSE-NEXT: por %xmm10, %xmm14 +; SSE-NEXT: packuswb %xmm14, %xmm10 +; SSE-NEXT: movdqa %xmm1, %xmm12 +; SSE-NEXT: pandn %xmm10, %xmm12 +; SSE-NEXT: pand %xmm1, %xmm13 +; SSE-NEXT: por %xmm13, %xmm12 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Reload +; SSE-NEXT: movdqa %xmm9, %xmm10 +; SSE-NEXT: pxor %xmm3, %xmm3 +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm3[8],xmm10[9],xmm3[9],xmm10[10],xmm3[10],xmm10[11],xmm3[11],xmm10[12],xmm3[12],xmm10[13],xmm3[13],xmm10[14],xmm3[14],xmm10[15],xmm3[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm3[0],xmm9[1],xmm3[1],xmm9[2],xmm3[2],xmm9[3],xmm3[3],xmm9[4],xmm3[4],xmm9[5],xmm3[5],xmm9[6],xmm3[6],xmm9[7],xmm3[7] +; SSE-NEXT: pxor %xmm4, %xmm4 +; SSE-NEXT: movdqa %xmm9, %xmm13 +; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[1,0],xmm10[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm9 = xmm9[2,0],xmm10[2,3] +; SSE-NEXT: psrlq $48, %xmm10 +; SSE-NEXT: psrldq {{.*#+}} xmm13 = xmm13[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm13 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm10 = xmm9[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm13, %xmm10 +; SSE-NEXT: movdqa %xmm15, %xmm13 +; SSE-NEXT: pandn %xmm10, %xmm13 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: movdqa %xmm3, %xmm10 +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm4[8],xmm10[9],xmm4[9],xmm10[10],xmm4[10],xmm10[11],xmm4[11],xmm10[12],xmm4[12],xmm10[13],xmm4[13],xmm10[14],xmm4[14],xmm10[15],xmm4[15] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5] +; SSE-NEXT: movdqa %xmm0, %xmm9 +; SSE-NEXT: pandn %xmm10, %xmm9 +; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm10[3,1,1,2,4,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm14 +; SSE-NEXT: por %xmm9, %xmm14 +; SSE-NEXT: packuswb %xmm14, %xmm14 +; SSE-NEXT: pand %xmm15, %xmm14 +; SSE-NEXT: por %xmm13, %xmm14 +; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; SSE-NEXT: movdqa %xmm3, %xmm9 +; SSE-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm3[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[0,1,2,0] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,7,4] +; SSE-NEXT: movdqa %xmm11, %xmm13 +; SSE-NEXT: pandn %xmm10, %xmm13 +; SSE-NEXT: punpcklbw {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3],xmm9[4],xmm4[4],xmm9[5],xmm4[5],xmm9[6],xmm4[6],xmm9[7],xmm4[7] +; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,3,1,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm9[0,1,1,3,4,5,6,7] +; SSE-NEXT: pand %xmm11, %xmm9 +; SSE-NEXT: por %xmm9, %xmm13 +; SSE-NEXT: packuswb %xmm13, %xmm9 +; SSE-NEXT: movdqa %xmm1, %xmm13 +; SSE-NEXT: pandn %xmm9, %xmm13 +; SSE-NEXT: pand %xmm1, %xmm14 +; SSE-NEXT: por %xmm14, %xmm13 +; SSE-NEXT: movdqa (%rsp), %xmm3 # 16-byte Reload +; SSE-NEXT: movdqa %xmm3, %xmm9 +; SSE-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15] +; SSE-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] +; SSE-NEXT: movdqa %xmm3, %xmm10 +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm9[0,0] +; SSE-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm9[2,3] +; SSE-NEXT: psrlq $48, %xmm9 +; SSE-NEXT: psrldq {{.*#+}} xmm10 = xmm10[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; SSE-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3] +; SSE-NEXT: pshuflw {{.*#+}} xmm9 = xmm3[3,1,2,3,4,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,1,0,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm9 = xmm9[0,1,2,3,4,4,5,7] +; SSE-NEXT: packuswb %xmm10, %xmm9 +; SSE-NEXT: movdqa %xmm6, %xmm10 +; SSE-NEXT: punpckhbw {{.*#+}} xmm10 = xmm10[8],xmm5[8],xmm10[9],xmm5[9],xmm10[10],xmm5[10],xmm10[11],xmm5[11],xmm10[12],xmm5[12],xmm10[13],xmm5[13],xmm10[14],xmm5[14],xmm10[15],xmm5[15] +; SSE-NEXT: pshufd {{.*#+}} xmm10 = xmm10[1,1,2,3] +; SSE-NEXT: pshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,5,5,5,5] +; SSE-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3],xmm6[4],xmm5[4],xmm6[5],xmm5[5],xmm6[6],xmm5[6],xmm6[7],xmm5[7] +; SSE-NEXT: pshufhw {{.*#+}} xmm14 = xmm6[0,1,2,3,7,5,6,7] +; SSE-NEXT: pshufd {{.*#+}} xmm14 = xmm14[0,2,2,3] +; SSE-NEXT: pshuflw {{.*#+}} xmm14 = xmm14[3,1,1,2,4,5,6,7] +; SSE-NEXT: pand %xmm0, %xmm14 +; SSE-NEXT: pandn %xmm10, %xmm0 +; SSE-NEXT: por %xmm14, %xmm0 ; SSE-NEXT: packuswb %xmm0, %xmm0 -; SSE-NEXT: pand %xmm13, %xmm0 -; SSE-NEXT: pandn %xmm8, %xmm13 -; SSE-NEXT: por %xmm13, %xmm0 -; SSE-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Reload -; SSE-NEXT: movdqa %xmm14, %xmm1 -; SSE-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1],xmm1[2],xmm5[2],xmm1[3],xmm5[3],xmm1[4],xmm5[4],xmm1[5],xmm5[5],xmm1[6],xmm5[6],xmm1[7],xmm5[7] -; SSE-NEXT: punpckhbw {{.*#+}} xmm14 = xmm14[8],xmm5[8],xmm14[9],xmm5[9],xmm14[10],xmm5[10],xmm14[11],xmm5[11],xmm14[12],xmm5[12],xmm14[13],xmm5[13],xmm14[14],xmm5[14],xmm14[15],xmm5[15] -; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,3,1,1] -; SSE-NEXT: pshuflw {{.*#+}} xmm1 = xmm1[0,1,1,3,4,5,6,7] -; SSE-NEXT: pand %xmm11, %xmm1 -; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm14[0,1,2,3,7,5,6,7] +; SSE-NEXT: pand %xmm15, %xmm0 +; SSE-NEXT: pandn %xmm9, %xmm15 +; SSE-NEXT: por %xmm15, %xmm0 +; SSE-NEXT: movdqa %xmm2, %xmm4 +; SSE-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7] +; SSE-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm5[8],xmm2[9],xmm5[9],xmm2[10],xmm5[10],xmm2[11],xmm5[11],xmm2[12],xmm5[12],xmm2[13],xmm5[13],xmm2[14],xmm5[14],xmm2[15],xmm5[15] +; SSE-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,3,1,1] +; SSE-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,1,1,3,4,5,6,7] +; SSE-NEXT: pand %xmm11, %xmm4 +; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm2[0,1,2,3,7,5,6,7] ; SSE-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,1,2,0] ; SSE-NEXT: pshufhw {{.*#+}} xmm5 = xmm5[0,1,2,3,5,5,7,4] ; SSE-NEXT: pandn %xmm5, %xmm11 -; SSE-NEXT: por %xmm1, %xmm11 -; SSE-NEXT: pand %xmm4, %xmm0 -; SSE-NEXT: packuswb %xmm11, %xmm1 -; SSE-NEXT: pandn %xmm1, %xmm4 -; SSE-NEXT: por %xmm0, %xmm4 +; SSE-NEXT: por %xmm4, %xmm11 +; SSE-NEXT: pand %xmm1, %xmm0 +; SSE-NEXT: packuswb %xmm11, %xmm4 +; SSE-NEXT: pandn %xmm4, %xmm1 +; SSE-NEXT: por %xmm0, %xmm1 ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE-NEXT: movaps %xmm0, 16(%rsi) ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload @@ -3606,11 +3577,11 @@ define void @load_i8_stride6_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, ptr ; SSE-NEXT: movaps {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; SSE-NEXT: movaps %xmm0, (%r9) ; SSE-NEXT: movq {{[0-9]+}}(%rsp), %rax -; SSE-NEXT: movdqa %xmm4, 16(%rax) -; SSE-NEXT: movdqa %xmm6, 32(%rax) +; SSE-NEXT: movdqa %xmm1, 16(%rax) +; SSE-NEXT: movdqa %xmm13, 32(%rax) ; SSE-NEXT: movdqa %xmm12, 48(%rax) -; SSE-NEXT: movdqa %xmm2, (%rax) -; SSE-NEXT: addq $808, %rsp # imm = 0x328 +; SSE-NEXT: movdqa %xmm8, (%rax) +; SSE-NEXT: addq $792, %rsp # imm = 0x318 ; SSE-NEXT: retq ; ; AVX1-ONLY-LABEL: load_i8_stride6_vf64: diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll index e6de73d602b7..16b5e8db76ef 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-3.ll @@ -652,8 +652,7 @@ define void @store_i16_stride3_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 ; AVX512F-NEXT: vmovdqa {{.*#+}} ymm2 = ; AVX512F-NEXT: vpermd %ymm0, %ymm2, %ymm2 -; AVX512F-NEXT: vmovdqa {{.*#+}} ymm3 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535] -; AVX512F-NEXT: vpandn %ymm2, %ymm3, %ymm2 +; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2 ; AVX512F-NEXT: vpshufb {{.*#+}} ymm3 = zero,zero,ymm0[10,11],zero,zero,zero,zero,ymm0[12,13],zero,zero,zero,zero,ymm0[14,15],zero,zero,zero,zero,ymm0[16,17],zero,zero,zero,zero,ymm0[18,19],zero,zero,zero,zero ; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 ; AVX512F-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm2 @@ -1218,7 +1217,7 @@ define void @store_i16_stride3_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm3[0,1,2,3],zmm4[4,5,6,7] ; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm4 = <5,5,u,6,6,u,7,7,u,8,8,u,9,9,u,10> ; AVX512F-NEXT: vpermd (%rdx), %zmm4, %zmm5 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm5 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm5 ; AVX512F-NEXT: vmovdqa 32(%rdi), %ymm3 ; AVX512F-NEXT: vmovdqa {{.*#+}} ymm6 = [128,128,128,128,12,13,128,128,128,128,14,15,128,128,128,128,16,17,128,128,128,128,18,19,128,128,128,128,20,21,128,128] ; AVX512F-NEXT: vpshufb %ymm6, %ymm3, %ymm3 @@ -2386,7 +2385,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm18 = <5,5,u,6,6,u,7,7,u,8,8,u,9,9,u,10> ; AVX512F-NEXT: vpermd 64(%rdx), %zmm18, %zmm10 ; AVX512F-NEXT: vmovdqa64 {{.*#+}} zmm22 = [0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535] -; AVX512F-NEXT: vpternlogq $184, %zmm0, %zmm22, %zmm10 +; AVX512F-NEXT: vpternlogd $184, %zmm0, %zmm22, %zmm10 ; AVX512F-NEXT: vmovdqa 96(%rdi), %ymm0 ; AVX512F-NEXT: vmovdqa %ymm6, %ymm2 ; AVX512F-NEXT: vpshufb %ymm6, %ymm0, %ymm0 @@ -2469,7 +2468,7 @@ define void @store_i16_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1 ; AVX512F-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7] ; AVX512F-NEXT: vpermd (%rdx), %zmm18, %zmm1 -; AVX512F-NEXT: vpternlogq $184, %zmm0, %zmm22, %zmm1 +; AVX512F-NEXT: vpternlogd $184, %zmm0, %zmm22, %zmm1 ; AVX512F-NEXT: vmovdqa64 %zmm1, 64(%rcx) ; AVX512F-NEXT: vmovdqa64 %zmm6, 128(%rcx) ; AVX512F-NEXT: vmovdqa64 %zmm5, 192(%rcx) diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll index 9f479b0ee993..413cd7d63d8c 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-5.ll @@ -714,7 +714,7 @@ define void @store_i16_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpbroadcastq (%r8), %ymm6 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm6, %zmm4 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm4 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm4 ; AVX512F-SLOW-NEXT: vpsrlq $48, %xmm3, %xmm3 ; AVX512F-SLOW-NEXT: vpunpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1] ; AVX512F-SLOW-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] @@ -753,7 +753,7 @@ define void @store_i16_stride5_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpbroadcastq (%r8), %ymm6 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,1,1,1] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm6, %zmm4 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm4 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm4 ; AVX512F-FAST-NEXT: vpsrlq $48, %xmm1, %xmm1 ; AVX512F-FAST-NEXT: vpunpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm1[1] ; AVX512F-FAST-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7] @@ -1396,7 +1396,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastq (%r8), %ymm6 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm7 = ymm4[0,1,1,1] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,14,15,u,u,u,u,u,u,u,u,16,17,u,u,u,u,u,u,u,u,18,19,u,u] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm7 = ymm0[1,1,2,2] ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2,3],ymm5[4],ymm7[5],ymm5[6],ymm7[7,8],ymm5[9],ymm7[10,11],ymm5[12],ymm7[13],ymm5[14],ymm7[15] @@ -1474,7 +1474,7 @@ define void @store_i16_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastq (%r8), %ymm6 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm4[0,1,1,1] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm5 = ymm1[u,u,u,u,u,u,u,u,14,15,u,u,u,u,u,u,u,u,16,17,u,u,u,u,u,u,u,u,18,19,u,u] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm0[1,1,2,2] ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm5 = ymm7[0],ymm5[1],ymm7[2,3],ymm5[4],ymm7[5],ymm5[6],ymm7[7,8],ymm5[9],ymm7[10,11],ymm5[12],ymm7[13],ymm5[14],ymm7[15] @@ -2776,7 +2776,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm13 = [4,5,2,3,2,3,0,1,10,11,8,9,4,5,6,7] ; AVX512F-SLOW-NEXT: vpshufb %xmm13, %xmm0, %xmm0 ; AVX512F-SLOW-NEXT: vmovdqa64 (%rdx), %ymm17 -; AVX512F-SLOW-NEXT: vmovdqa 32(%rdx), %ymm5 +; AVX512F-SLOW-NEXT: vmovdqa 32(%rdx), %ymm6 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm2 = ymm17[3,2,3,3,7,6,7,7] ; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %ymm1 ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm3 = ymm1[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15] @@ -2793,8 +2793,8 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm2 = xmm2[0,1,2,3,4,4,5,6] ; AVX512F-SLOW-NEXT: vmovdqa64 (%rdi), %ymm20 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm3 = ymm20[2,3,2,3,6,7,6,7] -; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %ymm6 -; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm7 = ymm6[0,1,2,3,7,6,5,7,8,9,10,11,15,14,13,15] +; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %ymm5 +; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm7 = ymm5[0,1,2,3,7,6,5,7,8,9,10,11,15,14,13,15] ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm7 = ymm7[2,3,2,2,6,7,6,6] ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm7[0],ymm3[1],ymm7[2],ymm3[3],ymm7[4,5],ymm3[6],ymm7[7,8],ymm3[9],ymm7[10],ymm3[11],ymm7[12,13],ymm3[14],ymm7[15] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,2,2] @@ -2804,7 +2804,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastq 24(%r8), %ymm0 ; AVX512F-SLOW-NEXT: vpbroadcastq 32(%r8), %ymm2 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm19 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm19 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm19 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rsi), %ymm3 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm2 = ; AVX512F-SLOW-NEXT: vpshufb %ymm2, %ymm3, %ymm0 @@ -2822,7 +2822,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm4 = ; AVX512F-SLOW-NEXT: vpshufb %ymm4, %ymm0, %ymm2 ; AVX512F-SLOW-NEXT: vmovdqa64 %ymm4, %ymm23 -; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm8 = ymm5[3,0,3,0,7,4,7,4] +; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm8 = ymm6[3,0,3,0,7,4,7,4] ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm8[0],ymm2[1],ymm8[2],ymm2[3],ymm8[4,5],ymm2[6],ymm8[7,8],ymm2[9],ymm8[10],ymm2[11],ymm8[12,13],ymm2[14],ymm8[15] ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm8 = <10,11,u,u,6,7,u,u,8,9,8,9,u,u,8,9> ; AVX512F-SLOW-NEXT: vpshufb %xmm8, %xmm10, %xmm10 @@ -2831,15 +2831,15 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm9 = ymm9[0,1,0,0] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm9, %zmm2 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm2 -; AVX512F-SLOW-NEXT: vmovdqa (%r8), %ymm10 -; AVX512F-SLOW-NEXT: vmovdqa 32(%r8), %ymm9 +; AVX512F-SLOW-NEXT: vmovdqa (%r8), %ymm9 +; AVX512F-SLOW-NEXT: vmovdqa 32(%r8), %ymm10 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm11 = [128,128,128,128,12,13,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128] -; AVX512F-SLOW-NEXT: vpshufb %ymm11, %ymm9, %ymm4 +; AVX512F-SLOW-NEXT: vpshufb %ymm11, %ymm10, %ymm4 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm21 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535] -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm9 = ymm9[0,1,1,1] -; AVX512F-SLOW-NEXT: vpandnq %ymm9, %ymm21, %ymm9 -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm9, %zmm9 -; AVX512F-SLOW-NEXT: vpternlogq $248, %zmm21, %zmm2, %zmm9 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm10 = ymm10[0,1,1,1] +; AVX512F-SLOW-NEXT: vpandnq %ymm10, %ymm21, %ymm10 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm10, %zmm10 +; AVX512F-SLOW-NEXT: vpternlogq $248, %zmm21, %zmm2, %zmm10 ; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %xmm2 ; AVX512F-SLOW-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm12[0],xmm2[0],xmm12[1],xmm2[1],xmm12[2],xmm2[2],xmm12[3],xmm2[3] ; AVX512F-SLOW-NEXT: vpshufb %xmm13, %xmm4, %xmm4 @@ -2860,43 +2860,44 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm7 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535] ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm2, %zmm7, %zmm4 ; AVX512F-SLOW-NEXT: vpbroadcastq (%r8), %ymm2 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm8 = ymm10[0,1,1,1] -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm8, %zmm2, %zmm2 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm2 -; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm4 = ymm18[0,1,2,1,4,5,6,5] -; AVX512F-SLOW-NEXT: vprolq $16, %ymm3, %ymm8 -; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm4 = ymm8[0,1],ymm4[2],ymm8[3],ymm4[4],ymm8[5,6],ymm4[7],ymm8[8,9],ymm4[10],ymm8[11],ymm4[12],ymm8[13,14],ymm4[15] -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3] -; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm8 = ymm18[2,3,2,3,6,7,6,7] +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm8 = ymm9[0,1,1,1] +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm8, %zmm2, %zmm12 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm12 +; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm2 = ymm18[0,1,2,1,4,5,6,5] +; AVX512F-SLOW-NEXT: vprolq $16, %ymm3, %ymm4 +; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0,1],ymm2[2],ymm4[3],ymm2[4],ymm4[5,6],ymm2[7],ymm4[8,9],ymm2[10],ymm4[11],ymm2[12],ymm4[13,14],ymm2[15] +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,3,2,3] +; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm4 = ymm18[2,3,2,3,6,7,6,7] ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm3 = ymm3[0,1,2,3,7,6,5,7,8,9,10,11,15,14,13,15] ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[2,3,2,2,6,7,6,6] -; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm8[1],ymm3[2],ymm8[3],ymm3[4,5],ymm8[6],ymm3[7,8],ymm8[9],ymm3[10],ymm8[11],ymm3[12,13],ymm8[14],ymm3[15] +; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4,5],ymm4[6],ymm3[7,8],ymm4[9],ymm3[10],ymm4[11],ymm3[12,13],ymm4[14],ymm3[15] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,2,2] -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm4 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 ; AVX512F-SLOW-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25,18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25] ; AVX512F-SLOW-NEXT: # ymm3 = mem[0,1,0,1] -; AVX512F-SLOW-NEXT: vpshufb %ymm3, %ymm0, %ymm8 -; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm12 = ymm5[1,1,1,2,5,5,5,6] -; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm8 = ymm8[0],ymm12[1],ymm8[2,3],ymm12[4],ymm8[5],ymm12[6],ymm8[7,8],ymm12[9],ymm8[10,11],ymm12[12],ymm8[13],ymm12[14],ymm8[15] -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,3,2,3] -; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm5 = ymm5[3,2,3,3,7,6,7,7] +; AVX512F-SLOW-NEXT: vpshufb %ymm3, %ymm0, %ymm4 +; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm8 = ymm6[1,1,1,2,5,5,5,6] +; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm8[1],ymm4[2,3],ymm8[4],ymm4[5],ymm8[6],ymm4[7,8],ymm8[9],ymm4[10,11],ymm8[12],ymm4[13],ymm8[14],ymm4[15] +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3] +; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm6 = ymm6[3,2,3,3,7,6,7,7] ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm0 = ymm0[0,1,2,3,5,6,7,7,8,9,10,11,13,14,15,15] ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[2,3,2,3,6,7,6,7] -; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm5[1],ymm0[2],ymm5[3,4],ymm0[5,6,7,8],ymm5[9],ymm0[10],ymm5[11,12],ymm0[13,14,15] +; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm6[1],ymm0[2],ymm6[3,4],ymm0[5,6,7,8],ymm6[9],ymm0[10],ymm6[11,12],ymm0[13,14,15] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,3,2] -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm8, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm4, %zmm7, %zmm0 -; AVX512F-SLOW-NEXT: vpbroadcastq 48(%r8), %ymm4 -; AVX512F-SLOW-NEXT: vpbroadcastq 56(%r8), %ymm5 -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm4 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm4, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm2, %zmm7, %zmm0 +; AVX512F-SLOW-NEXT: vpbroadcastq 48(%r8), %ymm2 +; AVX512F-SLOW-NEXT: vpbroadcastq 56(%r8), %ymm4 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm2, %zmm2 +; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0] +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm4, %zmm2 ; AVX512F-SLOW-NEXT: vmovdqa64 %ymm22, %ymm0 -; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm6, %ymm0 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm5 = ymm20[1,1,2,2] -; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm5[0],ymm0[1],ymm5[2,3],ymm0[4],ymm5[5],ymm0[6],ymm5[7,8],ymm0[9],ymm5[10,11],ymm0[12],ymm5[13],ymm0[14],ymm5[15] -; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm5 = ymm20[0,1,2,1,4,5,6,5] -; AVX512F-SLOW-NEXT: vprolq $16, %ymm6, %ymm6 -; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm5 = ymm6[0,1],ymm5[2],ymm6[3],ymm5[4],ymm6[5,6],ymm5[7],ymm6[8,9],ymm5[10],ymm6[11],ymm5[12],ymm6[13,14],ymm5[15] +; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm5, %ymm0 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm6 = ymm20[1,1,2,2] +; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm6[0],ymm0[1],ymm6[2,3],ymm0[4],ymm6[5],ymm0[6],ymm6[7,8],ymm0[9],ymm6[10,11],ymm0[12],ymm6[13],ymm0[14],ymm6[15] +; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm6 = ymm20[0,1,2,1,4,5,6,5] +; AVX512F-SLOW-NEXT: vprolq $16, %ymm5, %ymm5 +; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm5 = ymm5[0,1],ymm6[2],ymm5[3],ymm6[4],ymm5[5,6],ymm6[7],ymm5[8,9],ymm6[10],ymm5[11],ymm6[12],ymm5[13,14],ymm6[15] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,3,2,3] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqa64 %ymm23, %ymm5 @@ -2909,15 +2910,15 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm5, %zmm1 ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm0, %zmm16, %zmm1 -; AVX512F-SLOW-NEXT: vpshufb %ymm11, %ymm10, %ymm0 -; AVX512F-SLOW-NEXT: vpbroadcastq 16(%r8), %ymm3 -; AVX512F-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0 +; AVX512F-SLOW-NEXT: vpbroadcastq 16(%r8), %ymm0 +; AVX512F-SLOW-NEXT: vpandn %ymm0, %ymm4, %ymm0 +; AVX512F-SLOW-NEXT: vpshufb %ymm11, %ymm9, %ymm3 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0 ; AVX512F-SLOW-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, 64(%r9) -; AVX512F-SLOW-NEXT: vmovdqa64 %zmm4, 256(%r9) -; AVX512F-SLOW-NEXT: vmovdqa64 %zmm2, (%r9) -; AVX512F-SLOW-NEXT: vmovdqa64 %zmm9, 192(%r9) +; AVX512F-SLOW-NEXT: vmovdqa64 %zmm2, 256(%r9) +; AVX512F-SLOW-NEXT: vmovdqa64 %zmm12, (%r9) +; AVX512F-SLOW-NEXT: vmovdqa64 %zmm10, 192(%r9) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm19, 128(%r9) ; AVX512F-SLOW-NEXT: vzeroupper ; AVX512F-SLOW-NEXT: retq @@ -2927,7 +2928,7 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vmovdqa 32(%rsi), %ymm5 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm1 = ; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm5, %ymm0 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm1, %ymm23 +; AVX512F-FAST-NEXT: vmovdqa64 %ymm1, %ymm17 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %ymm7 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm1 = ymm7[1,1,2,2] ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5],ymm0[6],ymm1[7,8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13],ymm0[14],ymm1[15] @@ -2935,29 +2936,29 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vmovdqa 32(%rsi), %xmm2 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm12 = <6,7,u,u,10,11,6,7,u,u,8,9,u,u,12,13> ; AVX512F-FAST-NEXT: vpshufb %xmm12, %xmm2, %xmm1 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm19 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm20 ; AVX512F-FAST-NEXT: vpbroadcastq 40(%rdi), %xmm2 ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3],xmm2[4],xmm1[5],xmm2[6],xmm1[7] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; AVX512F-FAST-NEXT: vmovdqa (%rcx), %ymm4 +; AVX512F-FAST-NEXT: vmovdqa (%rcx), %ymm1 ; AVX512F-FAST-NEXT: vmovdqa 32(%rcx), %ymm8 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = -; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm8, %ymm1 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm2, %ymm24 -; AVX512F-FAST-NEXT: vmovdqa64 (%rdx), %ymm17 +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm3 = +; AVX512F-FAST-NEXT: vpshufb %ymm3, %ymm8, %ymm2 +; AVX512F-FAST-NEXT: vmovdqa64 %ymm3, %ymm24 +; AVX512F-FAST-NEXT: vmovdqa64 (%rdx), %ymm18 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdx), %ymm9 -; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm2 = ymm9[3,0,3,0,7,4,7,4] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4,5],ymm1[6],ymm2[7,8],ymm1[9],ymm2[10],ymm1[11],ymm2[12,13],ymm1[14],ymm2[15] +; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm4 = ymm9[3,0,3,0,7,4,7,4] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm4[0],ymm2[1],ymm4[2],ymm2[3],ymm4[4,5],ymm2[6],ymm4[7,8],ymm2[9],ymm4[10],ymm2[11],ymm4[12,13],ymm2[14],ymm4[15] ; AVX512F-FAST-NEXT: vmovdqa 32(%rcx), %xmm11 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm15 = <10,11,u,u,6,7,u,u,8,9,8,9,u,u,8,9> -; AVX512F-FAST-NEXT: vpshufb %xmm15, %xmm11, %xmm2 +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm4 = <10,11,u,u,6,7,u,u,8,9,8,9,u,u,8,9> +; AVX512F-FAST-NEXT: vpshufb %xmm4, %xmm11, %xmm13 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdx), %xmm6 -; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm13 = xmm6[1,2,2,2] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm2 = xmm2[0],xmm13[1],xmm2[2],xmm13[3],xmm2[4,5],xmm13[6],xmm2[7] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,0] -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 +; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm15 = xmm6[1,2,2,2] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm13 = xmm13[0],xmm15[1],xmm13[2],xmm15[3],xmm13[4,5],xmm15[6],xmm13[7] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,1,0,0] +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm13, %zmm15 +; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm15 ; AVX512F-FAST-NEXT: vmovdqa (%r8), %ymm0 ; AVX512F-FAST-NEXT: vmovdqa 32(%r8), %ymm2 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm13 = [128,128,128,128,12,13,128,128,128,128,128,128,128,128,14,15,128,128,128,128,128,128,128,128,16,17,128,128,128,128,128,128] @@ -2966,64 +2967,65 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm16 = [65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,1,1] ; AVX512F-FAST-NEXT: vpandnq %ymm2, %ymm16, %ymm2 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm18 -; AVX512F-FAST-NEXT: vpternlogq $248, %zmm16, %zmm1, %zmm18 -; AVX512F-FAST-NEXT: vmovdqa (%rcx), %xmm1 -; AVX512F-FAST-NEXT: vpshufb %xmm15, %xmm1, %xmm3 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm19 +; AVX512F-FAST-NEXT: vpternlogq $248, %zmm16, %zmm15, %zmm19 +; AVX512F-FAST-NEXT: vmovdqa (%rcx), %xmm3 +; AVX512F-FAST-NEXT: vpshufb %xmm4, %xmm3, %xmm4 ; AVX512F-FAST-NEXT: vmovdqa (%rdx), %xmm15 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} xmm14 = xmm15[1,2,2,2] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm14[1],xmm3[2],xmm14[3],xmm3[4,5],xmm14[6],xmm3[7] -; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm15[0],xmm1[1],xmm15[1],xmm1[2],xmm15[2],xmm1[3],xmm15[3] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm4 = xmm4[0],xmm14[1],xmm4[2],xmm14[3],xmm4[4,5],xmm14[6],xmm4[7] +; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm15[0],xmm3[1],xmm15[1],xmm3[2],xmm15[2],xmm3[3],xmm15[3] ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = [4,5,2,3,2,3,0,1,10,11,8,9,4,5,6,7] -; AVX512F-FAST-NEXT: vpshufb %xmm2, %xmm1, %xmm1 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm20 +; AVX512F-FAST-NEXT: vpshufb %xmm2, %xmm3, %xmm3 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm21 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm14 = [0,1,0,1,8,9,8,8] -; AVX512F-FAST-NEXT: vpermi2q %zmm3, %zmm1, %zmm14 -; AVX512F-FAST-NEXT: vpshufb %xmm12, %xmm10, %xmm1 -; AVX512F-FAST-NEXT: vpbroadcastq 8(%rdi), %xmm3 -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm3[1],xmm1[2,3],xmm3[4],xmm1[5],xmm3[6],xmm1[7] -; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm3 -; AVX512F-FAST-NEXT: vmovdqa64 32(%rdi), %xmm21 -; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3] +; AVX512F-FAST-NEXT: vpermi2q %zmm4, %zmm3, %zmm14 +; AVX512F-FAST-NEXT: vpshufb %xmm12, %xmm10, %xmm3 +; AVX512F-FAST-NEXT: vpbroadcastq 8(%rdi), %xmm4 +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} xmm3 = xmm3[0],xmm4[1],xmm3[2,3],xmm4[4],xmm3[5],xmm4[6],xmm3[7] +; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm4 +; AVX512F-FAST-NEXT: vmovdqa64 32(%rdi), %xmm22 +; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm4[0],xmm10[0],xmm4[1],xmm10[1],xmm4[2],xmm10[2],xmm4[3],xmm10[3] ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,2,3,8,9,10,11,4,5,4,5,6,7,12,13] -; AVX512F-FAST-NEXT: vpshufb %xmm2, %xmm3, %xmm3 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm22 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 -; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,1,0,1,4,5,4,5] +; AVX512F-FAST-NEXT: vpshufb %xmm2, %xmm4, %xmm4 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm23 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm4, %zmm3 +; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm3[0,1,0,1,4,5,4,5] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm16 = [65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm14, %zmm16, %zmm1 -; AVX512F-FAST-NEXT: vpbroadcastq (%r8), %ymm3 +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm14, %zmm16, %zmm3 +; AVX512F-FAST-NEXT: vpbroadcastq (%r8), %ymm4 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm10 = ymm0[0,1,1,1] -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm3, %zmm10 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm10 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm4, %zmm10 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm10 ; AVX512F-FAST-NEXT: vmovdqa (%rsi), %ymm15 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm23, %ymm1 -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm15, %ymm1 +; AVX512F-FAST-NEXT: vmovdqa64 %ymm17, %ymm2 +; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm15, %ymm3 ; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm14 -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm3 = ymm14[1,1,2,2] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm3[0],ymm1[1],ymm3[2,3],ymm1[4],ymm3[5],ymm1[6],ymm3[7,8],ymm1[9],ymm3[10,11],ymm1[12],ymm3[13],ymm1[14],ymm3[15] -; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm3 = ymm14[0,1,2,1,4,5,6,5] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm14[1,1,2,2] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm3 = ymm4[0],ymm3[1],ymm4[2,3],ymm3[4],ymm4[5],ymm3[6],ymm4[7,8],ymm3[9],ymm4[10,11],ymm3[12],ymm4[13],ymm3[14],ymm4[15] +; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm4 = ymm14[0,1,2,1,4,5,6,5] ; AVX512F-FAST-NEXT: vprolq $16, %ymm15, %ymm13 -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm3 = ymm13[0,1],ymm3[2],ymm13[3],ymm3[4],ymm13[5,6],ymm3[7],ymm13[8,9],ymm3[10],ymm13[11],ymm3[12],ymm13[13,14],ymm3[15] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,2,3] -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm3 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm24, %ymm1 -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm4, %ymm1 -; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm13 = ymm17[3,0,3,0,7,4,7,4] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm13 = ymm13[0],ymm1[1],ymm13[2],ymm1[3],ymm13[4,5],ymm1[6],ymm13[7,8],ymm1[9],ymm13[10],ymm1[11],ymm13[12,13],ymm1[14],ymm13[15] -; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25,18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25] -; AVX512F-FAST-NEXT: # ymm1 = mem[0,1,0,1] -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm4, %ymm2 -; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm12 = ymm17[1,1,1,2,5,5,5,6] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm4 = ymm13[0,1],ymm4[2],ymm13[3],ymm4[4],ymm13[5,6],ymm4[7],ymm13[8,9],ymm4[10],ymm13[11],ymm4[12],ymm13[13,14],ymm4[15] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,3,2,3] +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm3 +; AVX512F-FAST-NEXT: vmovdqa64 %ymm24, %ymm2 +; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm4 +; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm13 = ymm18[3,0,3,0,7,4,7,4] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm13 = ymm13[0],ymm4[1],ymm13[2],ymm4[3],ymm13[4,5],ymm4[6],ymm13[7,8],ymm4[9],ymm13[10],ymm4[11],ymm13[12,13],ymm4[14],ymm13[15] +; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25,18,19,0,0,22,23,22,23,0,0,20,21,0,0,24,25] +; AVX512F-FAST-NEXT: # ymm4 = mem[0,1,0,1] +; AVX512F-FAST-NEXT: vpshufb %ymm4, %ymm1, %ymm2 +; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm12 = ymm18[1,1,1,2,5,5,5,6] ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm12[1],ymm2[2,3],ymm12[4],ymm2[5],ymm12[6],ymm2[7,8],ymm12[9],ymm2[10,11],ymm12[12],ymm2[13],ymm12[14],ymm2[15] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,3,2,3] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm13, %zmm2 -; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm3, %zmm12, %zmm2 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm25, %ymm3 -; AVX512F-FAST-NEXT: vpshufb %ymm3, %ymm0, %ymm0 +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm17 = [65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535] +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm3, %zmm17, %zmm2 ; AVX512F-FAST-NEXT: vpbroadcastq 16(%r8), %ymm3 -; AVX512F-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm12 = [65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0] +; AVX512F-FAST-NEXT: vpandn %ymm3, %ymm12, %ymm3 +; AVX512F-FAST-NEXT: vmovdqa64 %ymm25, %ymm13 +; AVX512F-FAST-NEXT: vpshufb %ymm13, %ymm0, %ymm0 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0 ; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm13 = [30,31,28,29,26,27,30,31,30,31,28,29,30,31,28,29,30,31,28,29,26,27,30,31,30,31,28,29,30,31,28,29] @@ -3040,44 +3042,44 @@ define void @store_i16_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm8, %ymm3 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm7 = ymm9[3,2,3,3,7,6,7,7] ; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm7[1],ymm3[2],ymm7[3,4],ymm3[5,6,7,8],ymm7[9],ymm3[10],ymm7[11,12],ymm3[13,14,15] -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm8, %ymm1 +; AVX512F-FAST-NEXT: vpshufb %ymm4, %ymm8, %ymm4 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm7 = ymm9[1,1,1,2,5,5,5,6] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm7[1],ymm1[2,3],ymm7[4],ymm1[5],ymm7[6],ymm1[7,8],ymm7[9],ymm1[10,11],ymm7[12],ymm1[13],ymm7[14],ymm1[15] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm7[1],ymm4[2,3],ymm7[4],ymm4[5],ymm7[6],ymm4[7,8],ymm7[9],ymm4[10,11],ymm7[12],ymm4[13],ymm7[14],ymm4[15] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm7 = [2,3,2,3,10,10,11,10] -; AVX512F-FAST-NEXT: vpermi2q %zmm3, %zmm1, %zmm7 +; AVX512F-FAST-NEXT: vpermi2q %zmm3, %zmm4, %zmm7 ; AVX512F-FAST-NEXT: vpternlogq $226, %zmm5, %zmm16, %zmm7 -; AVX512F-FAST-NEXT: vpbroadcastq 48(%r8), %ymm1 -; AVX512F-FAST-NEXT: vpbroadcastq 56(%r8), %ymm3 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm1 -; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm11[0],xmm6[0],xmm11[1],xmm6[1],xmm11[2],xmm6[2],xmm11[3],xmm6[3] -; AVX512F-FAST-NEXT: vmovdqa64 %xmm20, %xmm5 -; AVX512F-FAST-NEXT: vpshufb %xmm5, %xmm3, %xmm3 -; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm4, %ymm2 -; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm4 = ymm17[3,2,3,3,7,6,7,7] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm4[1],ymm2[2],ymm4[3,4],ymm2[5,6,7,8],ymm4[9],ymm2[10],ymm4[11,12],ymm2[13,14,15] -; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [2,2,3,2,8,9,8,9] -; AVX512F-FAST-NEXT: vpermi2q %zmm3, %zmm2, %zmm4 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm19, %xmm2 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm21, %xmm3 -; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3] -; AVX512F-FAST-NEXT: vmovdqa64 %xmm22, %xmm3 -; AVX512F-FAST-NEXT: vpshufb %xmm3, %xmm2, %xmm2 -; AVX512F-FAST-NEXT: vpshufb %ymm13, %ymm15, %ymm3 +; AVX512F-FAST-NEXT: vpbroadcastq 48(%r8), %ymm3 +; AVX512F-FAST-NEXT: vpbroadcastq 56(%r8), %ymm4 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm3 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm7, %zmm12, %zmm3 +; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm11[0],xmm6[0],xmm11[1],xmm6[1],xmm11[2],xmm6[2],xmm11[3],xmm6[3] +; AVX512F-FAST-NEXT: vmovdqa64 %xmm21, %xmm5 +; AVX512F-FAST-NEXT: vpshufb %xmm5, %xmm4, %xmm4 +; AVX512F-FAST-NEXT: vpshufb %ymm2, %ymm1, %ymm1 +; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm2 = ymm18[3,2,3,3,7,6,7,7] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[2],ymm2[3,4],ymm1[5,6,7,8],ymm2[9],ymm1[10],ymm2[11,12],ymm1[13,14,15] +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm2 = [2,2,3,2,8,9,8,9] +; AVX512F-FAST-NEXT: vpermi2q %zmm4, %zmm1, %zmm2 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm20, %xmm1 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm22, %xmm4 +; AVX512F-FAST-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3] +; AVX512F-FAST-NEXT: vmovdqa64 %xmm23, %xmm4 +; AVX512F-FAST-NEXT: vpshufb %xmm4, %xmm1, %xmm1 +; AVX512F-FAST-NEXT: vpshufb %ymm13, %ymm15, %ymm4 ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm5 = ymm14[2,3,2,3,6,7,6,7] -; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm5[1],ymm3[2],ymm5[3],ymm3[4,5],ymm5[6],ymm3[7,8],ymm5[9],ymm3[10],ymm5[11],ymm3[12,13],ymm5[14],ymm3[15] +; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm4 = ymm4[0],ymm5[1],ymm4[2],ymm5[3],ymm4[4,5],ymm5[6],ymm4[7,8],ymm5[9],ymm4[10],ymm5[11],ymm4[12,13],ymm5[14],ymm4[15] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm5 = [2,3,2,2,8,9,8,9] -; AVX512F-FAST-NEXT: vpermi2q %zmm2, %zmm3, %zmm5 -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm4, %zmm12, %zmm5 -; AVX512F-FAST-NEXT: vpbroadcastq 24(%r8), %ymm2 -; AVX512F-FAST-NEXT: vpbroadcastq 32(%r8), %ymm3 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm2 -; AVX512F-FAST-NEXT: vmovdqa64 %zmm2, 128(%r9) -; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, 256(%r9) +; AVX512F-FAST-NEXT: vpermi2q %zmm1, %zmm4, %zmm5 +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm2, %zmm17, %zmm5 +; AVX512F-FAST-NEXT: vpbroadcastq 24(%r8), %ymm1 +; AVX512F-FAST-NEXT: vpbroadcastq 32(%r8), %ymm2 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm1 +; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, 128(%r9) +; AVX512F-FAST-NEXT: vmovdqa64 %zmm3, 256(%r9) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm0, 64(%r9) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm10, (%r9) -; AVX512F-FAST-NEXT: vmovdqa64 %zmm18, 192(%r9) +; AVX512F-FAST-NEXT: vmovdqa64 %zmm19, 192(%r9) ; AVX512F-FAST-NEXT: vzeroupper ; AVX512F-FAST-NEXT: retq ; @@ -5924,12 +5926,12 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastq 96(%r8), %ymm2 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm18 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm31, %zmm18, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm31, %zmm18, %zmm1 ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm25, %zmm19, %zmm27 ; AVX512F-SLOW-NEXT: vpbroadcastq 24(%r8), %ymm2 ; AVX512F-SLOW-NEXT: vpbroadcastq 32(%r8), %ymm25 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm25, %zmm2, %zmm2 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm27, %zmm18, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm27, %zmm18, %zmm2 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0] ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm31 # 64-byte Reload ; AVX512F-SLOW-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm31 # 64-byte Folded Reload @@ -5972,28 +5974,28 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastq 64(%r8), %ymm18 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm8, %zmm18, %zmm8 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm22, %zmm18, %zmm8 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm22, %zmm18, %zmm8 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm28, %zmm27, %zmm22 ; AVX512F-SLOW-NEXT: vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm21 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm21 = mem[0,1,0,1,4,5,4,5] ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm22, %zmm24, %zmm21 ; AVX512F-SLOW-NEXT: vpbroadcastq (%r8), %ymm22 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm22, %zmm10 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm21, %zmm18, %zmm10 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm21, %zmm18, %zmm10 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm13, %zmm14, %zmm13 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm11, %zmm12, %zmm11 ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm13, %zmm24, %zmm11 ; AVX512F-SLOW-NEXT: vpbroadcastq 112(%r8), %ymm12 ; AVX512F-SLOW-NEXT: vpbroadcastq 120(%r8), %ymm13 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm13, %zmm12, %zmm12 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm11, %zmm16, %zmm12 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm11, %zmm16, %zmm12 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm7, %zmm15, %zmm7 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm6, %zmm4 ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm7, %zmm24, %zmm4 ; AVX512F-SLOW-NEXT: vpbroadcastq 48(%r8), %ymm6 ; AVX512F-SLOW-NEXT: vpbroadcastq 56(%r8), %ymm7 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm4, %zmm16, %zmm6 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm4, %zmm16, %zmm6 ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm30, %zmm19, %zmm9 ; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm20, %zmm19, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535] @@ -6186,7 +6188,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastq 112(%r8), %ymm0 ; AVX512F-FAST-NEXT: vpbroadcastq 120(%r8), %ymm1 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm16 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm2, %zmm17, %zmm16 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm2, %zmm17, %zmm16 ; AVX512F-FAST-NEXT: vmovdqa 64(%rsi), %ymm8 ; AVX512F-FAST-NEXT: vpshufb %ymm12, %ymm8, %ymm0 ; AVX512F-FAST-NEXT: vmovdqa64 64(%rdi), %ymm24 @@ -6231,7 +6233,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastq 48(%r8), %ymm6 ; AVX512F-FAST-NEXT: vpbroadcastq 56(%r8), %ymm10 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm6, %zmm6 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm7, %zmm17, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm7, %zmm17, %zmm6 ; AVX512F-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm7 # 16-byte Reload ; AVX512F-FAST-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload ; AVX512F-FAST-NEXT: # xmm7 = xmm7[0],mem[0],xmm7[1],mem[1],xmm7[2],mem[2],xmm7[3],mem[3] @@ -6267,7 +6269,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastq 96(%r8), %ymm5 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm0 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm5 = [0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm8, %zmm5, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm8, %zmm5, %zmm0 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %xmm7 ; AVX512F-FAST-NEXT: vpunpcklwd {{[-0-9]+}}(%r{{[sb]}}p), %xmm7, %xmm7 # 16-byte Folded Reload ; AVX512F-FAST-NEXT: # xmm7 = xmm7[0],mem[0],xmm7[1],mem[1],xmm7[2],mem[2],xmm7[3],mem[3] @@ -6282,7 +6284,7 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastq 24(%r8), %ymm4 ; AVX512F-FAST-NEXT: vpbroadcastq 32(%r8), %ymm7 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm4, %zmm4 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm8, %zmm5, %zmm4 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm8, %zmm5, %zmm4 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm5 = [65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0,0,65535,65535,65535,0] ; AVX512F-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload ; AVX512F-FAST-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm3 # 64-byte Folded Reload @@ -6294,10 +6296,10 @@ define void @store_i16_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastq 64(%r8), %ymm5 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm5, %zmm2 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm20, %zmm5, %zmm2 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm20, %zmm5, %zmm2 ; AVX512F-FAST-NEXT: vpbroadcastq (%r8), %ymm7 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm7, %zmm7 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm22, %zmm5, %zmm7 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm22, %zmm5, %zmm7 ; AVX512F-FAST-NEXT: vpternlogq $226, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm26 # 64-byte Folded Reload ; AVX512F-FAST-NEXT: vpternlogq $226, %zmm25, %zmm1, %zmm12 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535,0,65535,65535,65535,65535] diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll index 7133214d9a80..b2c0e00825e6 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-6.ll @@ -3486,19 +3486,19 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-ONLY-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm19, %zmm18, %zmm9 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm10 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm17, %zmm10, %zmm9 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm17, %zmm10, %zmm9 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm22, %zmm21, %zmm11 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm20, %zmm10, %zmm11 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm20, %zmm10, %zmm11 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm24, %zmm16, %zmm10 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm12 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm23, %zmm12, %zmm10 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm23, %zmm12, %zmm10 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm8, %zmm3 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm12, %zmm3 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm12, %zmm3 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm1 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm6 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm4, %zmm6, %zmm1 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm4, %zmm6, %zmm1 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm5, %zmm2 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm6, %zmm2 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm6, %zmm2 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm2, (%rax) ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm1, 192(%rax) ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm3, 256(%rax) @@ -3695,23 +3695,23 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-ONLY-FAST-NEXT: vpermt2q %zmm3, %zmm5, %zmm2 ; AVX512F-ONLY-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm0[0,1,2,3] ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm2 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm2 ; AVX512F-ONLY-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm2, 256(%rax) ; AVX512F-ONLY-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm18[0,1,2,3],zmm10[0,1,2,3] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm8 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm8 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm8, 64(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm22, %zmm0, %zmm14 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm22, %zmm0, %zmm14 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm14, (%rax) -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm19, %zmm0, %zmm1 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm19, %zmm0, %zmm1 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm1, 192(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm17, %zmm0, %zmm1 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm17, %zmm0, %zmm1 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm1, 128(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm16, %zmm0, %zmm1 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm16, %zmm0, %zmm1 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm1, 320(%rax) ; AVX512F-ONLY-FAST-NEXT: popq %rax ; AVX512F-ONLY-FAST-NEXT: vzeroupper @@ -3935,19 +3935,19 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512DQ-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm19, %zmm26, %zmm6 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm7 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm6 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7, %zmm6 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm18, %zmm22, %zmm8 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm21, %zmm7, %zmm8 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm21, %zmm7, %zmm8 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm25, %zmm24, %zmm7 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm9 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm20, %zmm9, %zmm7 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm20, %zmm9, %zmm7 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm10 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm23, %zmm9, %zmm10 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm23, %zmm9, %zmm10 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm3, %zmm1 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm2, %zmm3, %zmm1 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm2 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm2 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm2 ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm2, 256(%rax) ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm1, 64(%rax) ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm8, (%rax) @@ -4154,22 +4154,22 @@ define void @store_i16_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,0,2,1,4,5,6,7] ; AVX512DQ-FAST-NEXT: vpermt2q %zmm5, %zmm9, %zmm6 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm5 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm4, %zmm5, %zmm6 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm4, %zmm5, %zmm6 ; AVX512DQ-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm6, (%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm3, %zmm5, %zmm7 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm3, %zmm5, %zmm7 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm7, 192(%rax) ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm1, %zmm3, %zmm0 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm1, %zmm3, %zmm0 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm0, 128(%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm18, %zmm3, %zmm2 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm18, %zmm3, %zmm2 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm2, 320(%rax) ; AVX512DQ-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm20[0,1,2,3],zmm17[0,1,2,3] ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm1, %zmm13 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm1, %zmm13 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm13, 256(%rax) ; AVX512DQ-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm19[0,1,2,3],zmm23[0,1,2,3] -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm1, %zmm12 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm1, %zmm12 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm12, 64(%rax) ; AVX512DQ-FAST-NEXT: vzeroupper ; AVX512DQ-FAST-NEXT: retq @@ -7763,10 +7763,10 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm6 # 32-byte Folded Reload ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm29 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm6 # 64-byte Folded Reload +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm6 # 64-byte Folded Reload ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2, %zmm23 # 32-byte Folded Reload -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm23 # 64-byte Folded Reload +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm23 # 64-byte Folded Reload ; AVX512F-ONLY-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm22, %zmm2, %zmm12 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm14, %zmm0, %zmm14 @@ -7862,27 +7862,27 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-ONLY-SLOW-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,2,2,3,4,5,6,7] ; AVX512F-ONLY-SLOW-NEXT: vpshufhw {{.*#+}} xmm11 = xmm11[0,1,2,3,4,4,4,4] ; AVX512F-ONLY-SLOW-NEXT: vpermq {{.*#+}} ymm11 = ymm11[0,0,2,1] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm12 # 64-byte Folded Reload +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm12 # 64-byte Folded Reload ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm16, %zmm20, %zmm4 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm4 # 64-byte Folded Reload +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm4 # 64-byte Folded Reload ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm5, %zmm5 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm10 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm5 # 64-byte Folded Reload +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm5 # 64-byte Folded Reload ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm17, %zmm24, %zmm16 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm27, %zmm10, %zmm16 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm27, %zmm10, %zmm16 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm13, %zmm8, %zmm8 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm25, %zmm10, %zmm8 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm25, %zmm10, %zmm8 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm14, %zmm2 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm22, %zmm10, %zmm2 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm22, %zmm10, %zmm2 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm19, %zmm18, %zmm10 ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm13 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm28, %zmm13, %zmm10 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm28, %zmm13, %zmm10 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm7, %zmm7 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm30, %zmm13, %zmm7 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm30, %zmm13, %zmm7 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm15, %zmm3, %zmm3 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm21, %zmm13, %zmm3 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm21, %zmm13, %zmm3 ; AVX512F-ONLY-SLOW-NEXT: vinserti64x4 $1, %ymm11, %zmm1, %zmm1 -; AVX512F-ONLY-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm13, %zmm1 +; AVX512F-ONLY-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm13, %zmm1 ; AVX512F-ONLY-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512F-ONLY-SLOW-NEXT: vmovdqa64 %zmm3, 192(%rax) @@ -8295,40 +8295,40 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-ONLY-FAST-NEXT: vpermt2q %zmm7, %zmm27, %zmm6 ; AVX512F-ONLY-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm18[0,1,2,3] ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm7 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm7, %zmm6 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm0, %zmm7, %zmm6 ; AVX512F-ONLY-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm6, 256(%rax) ; AVX512F-ONLY-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm20[0,1,2,3] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm7, %zmm9 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm0, %zmm7, %zmm9 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm9, 448(%rax) ; AVX512F-ONLY-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm5[0,1,2,3],zmm28[0,1,2,3] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm7, %zmm8 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm0, %zmm7, %zmm8 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm8, 640(%rax) ; AVX512F-ONLY-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm2[0,1,2,3],zmm17[0,1,2,3] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm0, %zmm7, %zmm3 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm0, %zmm7, %zmm3 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm3, 64(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm16, %zmm0, %zmm12 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm16, %zmm0, %zmm12 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm12, (%rax) -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm26, %zmm0, %zmm4 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm26, %zmm0, %zmm4 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm4, 192(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, %zmm25, %zmm1, %zmm10 +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, %zmm25, %zmm1, %zmm10 ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm10, 128(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm2 # 64-byte Folded Reload +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm2 # 64-byte Folded Reload ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm2, 320(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm2, 384(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 64-byte Folded Reload ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm2, 576(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm0, 512(%rax) ; AVX512F-ONLY-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload -; AVX512F-ONLY-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload +; AVX512F-ONLY-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm0 # 64-byte Folded Reload ; AVX512F-ONLY-FAST-NEXT: vmovdqa64 %zmm0, 704(%rax) ; AVX512F-ONLY-FAST-NEXT: addq $1256, %rsp # imm = 0x4E8 ; AVX512F-ONLY-FAST-NEXT: vzeroupper @@ -8731,10 +8731,10 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512DQ-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm2 # 32-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm26 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm2 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm2 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm3 # 32-byte Folded Reload -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm3 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm3 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm4 # 32-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload @@ -8807,27 +8807,27 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512DQ-SLOW-NEXT: vpermq {{.*#+}} ymm13 = ymm13[0,1,0,1] ; AVX512DQ-SLOW-NEXT: vpshuflw {{.*#+}} ymm10 = ymm10[0,0,2,1,4,5,6,7,8,8,10,9,12,13,14,15] ; AVX512DQ-SLOW-NEXT: vpermq {{.*#+}} ymm10 = ymm10[2,2,2,2] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm4 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm4 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm31, %zmm30, %zmm25 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm25 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm25 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm18, %zmm23, %zmm6 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm18 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm6 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm6 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm21, %zmm17, %zmm16 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm16 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm16 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm14, %zmm24, %zmm14 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm14 # 64-byte Folded Reload +; AVX512DQ-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm18, %zmm14 # 64-byte Folded Reload ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm5, %zmm5 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm29, %zmm18, %zmm5 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm29, %zmm18, %zmm5 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm8, %zmm8 ; AVX512DQ-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm9 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm19, %zmm9, %zmm8 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm19, %zmm9, %zmm8 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm15, %zmm0 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm20, %zmm9, %zmm0 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm20, %zmm9, %zmm0 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm11, %zmm7, %zmm7 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm22, %zmm9, %zmm7 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm22, %zmm9, %zmm7 ; AVX512DQ-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm13, %zmm10 -; AVX512DQ-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm9, %zmm10 +; AVX512DQ-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm9, %zmm10 ; AVX512DQ-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm10, 256(%rax) ; AVX512DQ-SLOW-NEXT: vmovdqa64 %zmm7, 448(%rax) @@ -9253,43 +9253,43 @@ define void @store_i16_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512DQ-FAST-NEXT: vpshuflw {{.*#+}} xmm13 = xmm6[0,0,2,1,4,5,6,7] ; AVX512DQ-FAST-NEXT: vpermt2q %zmm11, %zmm18, %zmm13 ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm11 = [65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm9, %zmm11, %zmm13 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm9, %zmm11, %zmm13 ; AVX512DQ-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm13, (%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm5, %zmm11, %zmm10 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm5, %zmm11, %zmm10 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm10, 192(%rax) ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm5 = [65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0] -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm5, %zmm8 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm5, %zmm8 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm8, 128(%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm29, %zmm5, %zmm3 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm29, %zmm5, %zmm3 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm3, 320(%rax) ; AVX512DQ-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm20[0,1,2,3],zmm25[0,1,2,3] ; AVX512DQ-FAST-NEXT: vmovdqa64 {{.*#+}} zmm3 = [65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] ; AVX512DQ-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm6 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm6 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm6, 256(%rax) ; AVX512DQ-FAST-NEXT: vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm0 # 64-byte Folded Reload ; AVX512DQ-FAST-NEXT: # zmm0 = zmm17[0,1,2,3],mem[0,1,2,3] ; AVX512DQ-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm6 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm6 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm6, 448(%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm12, %zmm11, %zmm4 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm12, %zmm11, %zmm4 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm4, 384(%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm7 # 64-byte Folded Reload +; AVX512DQ-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm7 # 64-byte Folded Reload ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm7, 576(%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm2 # 64-byte Folded Reload +; AVX512DQ-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm2 # 64-byte Folded Reload ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm2, 512(%rax) -; AVX512DQ-FAST-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm1 # 64-byte Folded Reload +; AVX512DQ-FAST-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm5, %zmm1 # 64-byte Folded Reload ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm1, 704(%rax) ; AVX512DQ-FAST-NEXT: vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm0 # 64-byte Folded Reload ; AVX512DQ-FAST-NEXT: # zmm0 = zmm26[0,1,2,3],mem[0,1,2,3] ; AVX512DQ-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm1 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm1 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm1, 640(%rax) ; AVX512DQ-FAST-NEXT: vshufi64x2 $68, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22, %zmm0 # 64-byte Folded Reload ; AVX512DQ-FAST-NEXT: # zmm0 = zmm22[0,1,2,3],mem[0,1,2,3] ; AVX512DQ-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload -; AVX512DQ-FAST-NEXT: vpternlogq $184, %zmm0, %zmm3, %zmm1 +; AVX512DQ-FAST-NEXT: vpternlogd $184, %zmm0, %zmm3, %zmm1 ; AVX512DQ-FAST-NEXT: vmovdqa64 %zmm1, 64(%rax) ; AVX512DQ-FAST-NEXT: addq $1224, %rsp # imm = 0x4C8 ; AVX512DQ-FAST-NEXT: vzeroupper diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll index 5934b80893ce..0bb8e17e0ee4 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-7.ll @@ -458,7 +458,7 @@ define void @store_i16_stride7_vf4(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[4,5,12,13,u,u,u,u,u,u],zero,zero,zero,zero,ymm3[6,7],zero,zero,ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 ; AVX512F-SLOW-NEXT: vporq %zmm1, %zmm2, %zmm1 -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 ; AVX512F-SLOW-NEXT: vextracti32x4 $2, %zmm1, 32(%rax) ; AVX512F-SLOW-NEXT: vextracti32x4 $3, %zmm1, %xmm0 ; AVX512F-SLOW-NEXT: vmovq %xmm0, 48(%rax) @@ -1059,7 +1059,7 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpbroadcastd 4(%r10), %ymm12 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm11 ; AVX512F-SLOW-NEXT: vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm11 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm11 ; AVX512F-SLOW-NEXT: vpsrlq $48, %xmm4, %xmm4 ; AVX512F-SLOW-NEXT: vpunpckhqdq {{.*#+}} xmm3 = xmm3[1],xmm4[1] ; AVX512F-SLOW-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] @@ -1141,7 +1141,7 @@ define void @store_i16_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpbroadcastd 4(%r10), %ymm5 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4 ; AVX512F-FAST-NEXT: vpternlogq $236, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm4 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm4 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm4 ; AVX512F-FAST-NEXT: vmovdqa %xmm0, 96(%rax) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm4, (%rax) ; AVX512F-FAST-NEXT: vmovdqa %ymm1, 64(%rax) @@ -2341,8 +2341,7 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm10 = ymm2[u,u,u,u,u,u,u,u,u,u,14,15],zero,zero,ymm2[u,u,u,u,u,u,u,u,u,u,16,17],zero,zero,ymm2[u,u,u,u] ; AVX512F-SLOW-NEXT: vporq %ymm5, %ymm10, %ymm19 ; AVX512F-SLOW-NEXT: vpbroadcastd 8(%rax), %ymm5 -; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm10 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535] -; AVX512F-SLOW-NEXT: vpandn %ymm5, %ymm10, %ymm5 +; AVX512F-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm5 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm10 = ymm13[12,13,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm13[14,15,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm13[16,17,u,u] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm5, %zmm5 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm10 = ymm6[0,1,1,3,4,5,5,7] @@ -2427,22 +2426,22 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm16, %zmm7, %zmm7 ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rcx ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm17, %zmm9, %zmm9 -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm9 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm9 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm19, %zmm4, %zmm4 ; AVX512F-SLOW-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm5 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm5 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm11, %zmm10, %zmm4 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm7 = zmm20[2,2,2,3,6,6,6,7] -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm7 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm7 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm12, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm3 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm0 = zmm1[0,0,1,1,4,4,5,5] -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm0 ; AVX512F-SLOW-NEXT: vpbroadcastd (%rax), %ymm1 ; AVX512F-SLOW-NEXT: vpbroadcastd 4(%rax), %ymm4 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm1 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm1 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm6 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm13[2,1,3,2] @@ -2556,20 +2555,20 @@ define void @store_i16_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rcx ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm16, %zmm3, %zmm2 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm17, %zmm0, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm18, %zmm8, %zmm2 ; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm10 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm10 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm0 = zmm19[2,2,2,3,6,6,6,7] -; AVX512F-FAST-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm6 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm6 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm0 = zmm11[0,0,1,1,4,4,5,5] -; AVX512F-FAST-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm0 ; AVX512F-FAST-NEXT: vpbroadcastd (%rax), %ymm2 ; AVX512F-FAST-NEXT: vpbroadcastd 4(%rax), %ymm3 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm2 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm2 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm4 ; AVX512F-FAST-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm1 @@ -5262,12 +5261,12 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm0 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm9 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm9 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm3 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm3 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm7 = ymm15[2,1,3,2] ; AVX512F-SLOW-NEXT: vpbroadcastd 32(%rax), %ymm9 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm7, %zmm7 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm7 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm7 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm7 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm8, %zmm17, %zmm1 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm13, %zmm18, %zmm8 @@ -5276,7 +5275,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd 40(%rax), %ymm9 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm1, %zmm1 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm9 # 64-byte Reload -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm1 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm1 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Reload ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm19, %zmm8, %zmm8 @@ -5295,7 +5294,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd 4(%rax), %ymm10 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm9, %zmm9 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Reload -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm9 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm9 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm9 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm14, %zmm2 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4 @@ -5303,7 +5302,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vbroadcasti64x4 {{.*#+}} zmm2 = [0,5,4,0,0,6,5,0,0,5,4,0,0,6,5,0] ; AVX512F-SLOW-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3] ; AVX512F-SLOW-NEXT: vpermd (%rax), %zmm2, %zmm2 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm2 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm2 ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm2, 128(%rax) @@ -5541,11 +5540,11 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vbroadcasti64x4 {{.*#+}} zmm7 = [30,5,0,0,31,6,0,31,30,5,0,0,31,6,0,31] ; AVX512F-FAST-NEXT: # zmm7 = mem[0,1,2,3,0,1,2,3] ; AVX512F-FAST-NEXT: vpermi2d %zmm3, %zmm8, %zmm7 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm7 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm7 ; AVX512F-FAST-NEXT: vpbroadcastd 36(%rax), %ymm6 ; AVX512F-FAST-NEXT: vpbroadcastd 40(%rax), %ymm8 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm8, %zmm6, %zmm6 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm6 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm8 = [65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535,65535,0,0,65535,65535,65535,65535] ; AVX512F-FAST-NEXT: vpternlogq $226, %zmm26, %zmm8, %zmm5 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 @@ -5558,7 +5557,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm11 = ymm19[0,0,1,3] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm12 = ymm18[2,2,2,3] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm13 = ymm17[2,1,3,2] -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm5 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm27, %zmm5 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm5 ; AVX512F-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Reload ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm15[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u] @@ -5574,7 +5573,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpermd %ymm9, %ymm0, %ymm0 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm3, %zmm3 ; AVX512F-FAST-NEXT: vpermd %zmm3, %zmm21, %zmm3 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm3 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm28, %zmm3 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm14, %zmm3 ; AVX512F-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm4 # 32-byte Folded Reload ; AVX512F-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11, %zmm9 # 32-byte Folded Reload @@ -5591,7 +5590,7 @@ define void @store_i16_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm16 ; AVX512F-FAST-NEXT: vpbroadcastd 32(%rax), %ymm2 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm22, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm22, %zmm0 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm20, %zmm0 ; AVX512F-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-FAST-NEXT: vmovdqa64 %zmm3, 128(%rax) @@ -10946,7 +10945,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vbroadcasti64x4 {{.*#+}} zmm11 = [0,5,4,0,0,6,5,0,0,5,4,0,0,6,5,0] ; AVX512F-SLOW-NEXT: # zmm11 = mem[0,1,2,3,0,1,2,3] ; AVX512F-SLOW-NEXT: vpermd 64(%rax), %zmm11, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm17, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm17, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm0 = ymm6[3,3,3,3,7,7,7,7] ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm1 = ymm3[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15] @@ -10973,7 +10972,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd 96(%rax), %ymm5 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm5 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm5, %zmm4 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm2, %zmm5, %zmm4 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm5, %zmm15 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-SLOW-NEXT: vmovdqa 96(%rsi), %xmm2 @@ -11007,7 +11006,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd 100(%rax), %ymm2 ; AVX512F-SLOW-NEXT: vpbroadcastd 104(%rax), %ymm3 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm16, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm16, %zmm2 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-SLOW-NEXT: vmovdqa 64(%rcx), %xmm1 ; AVX512F-SLOW-NEXT: vmovdqa 64(%rdx), %xmm2 @@ -11049,7 +11048,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd 68(%rax), %ymm5 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm4, %zmm4 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm9 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm9, %zmm4 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm9, %zmm4 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %xmm1 ; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %xmm5 @@ -11085,7 +11084,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd (%rax), %ymm2 ; AVX512F-SLOW-NEXT: vpbroadcastd 4(%rax), %ymm3 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm2 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm9, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm9, %zmm2 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-SLOW-NEXT: vmovdqa64 %ymm23, %ymm7 ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} ymm1 = ymm7[1,2,3,3,4,5,6,7,9,10,11,11,12,13,14,15] @@ -11120,7 +11119,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1],ymm6[2],ymm1[3,4],ymm6[5],ymm1[6,7,8,9],ymm6[10],ymm1[11,12],ymm6[13],ymm1[14,15] ; AVX512F-SLOW-NEXT: vmovdqa64 %ymm1, %ymm21 ; AVX512F-SLOW-NEXT: vpermd (%rax), %zmm11, %zmm1 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm17, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm2, %zmm17, %zmm1 ; AVX512F-SLOW-NEXT: vmovdqu64 %zmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm1 = ymm10[3,3,3,3,7,7,7,7] ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm2 = ymm8[0,1,2,3,7,6,6,7,8,9,10,11,15,14,14,15] @@ -11145,7 +11144,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm11 = ymm11[2,1,3,2] ; AVX512F-SLOW-NEXT: vpbroadcastd 32(%rax), %ymm12 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm25 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm1, %zmm15, %zmm25 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm1, %zmm15, %zmm25 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %xmm1 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rsi), %xmm12 ; AVX512F-SLOW-NEXT: vprold $16, %xmm12, %xmm15 @@ -11178,7 +11177,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vpbroadcastd 36(%rax), %ymm0 ; AVX512F-SLOW-NEXT: vpbroadcastd 40(%rax), %ymm6 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm6, %zmm0, %zmm28 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm28 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm28 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rcx), %xmm6 ; AVX512F-SLOW-NEXT: vmovdqa64 %xmm29, %xmm0 ; AVX512F-SLOW-NEXT: vpshufb %xmm0, %xmm6, %xmm2 @@ -11284,9 +11283,9 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm2[4,5,6,7] ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm23 # 64-byte Reload ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm1, %zmm23 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm1, %zmm23 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm29 # 64-byte Reload -; AVX512F-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm29 # 64-byte Folded Reload +; AVX512F-SLOW-NEXT: vpternlogd $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm29 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535] ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm17 # 64-byte Reload ; AVX512F-SLOW-NEXT: vpternlogq $184, {{[-0-9]+}}(%r{{[sb]}}p), %zmm0, %zmm17 # 64-byte Folded Reload @@ -11643,7 +11642,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vmovdqa 32(%rax), %ymm0 ; AVX512F-FAST-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-FAST-NEXT: vpermi2d %zmm0, %zmm19, %zmm2 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm3, %zmm22, %zmm2 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm3, %zmm22, %zmm2 ; AVX512F-FAST-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm13 # 32-byte Reload ; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm13, %ymm0 @@ -11679,7 +11678,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vbroadcasti64x4 {{.*#+}} zmm2 = [0,13,4,0,0,14,5,0,0,13,4,0,0,14,5,0] ; AVX512F-FAST-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3] ; AVX512F-FAST-NEXT: vpermd %zmm0, %zmm2, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm3, %zmm25, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm3, %zmm25, %zmm0 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm0 ; AVX512F-FAST-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill ; AVX512F-FAST-NEXT: vpshufb %ymm11, %ymm1, %ymm0 @@ -11776,7 +11775,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastd 68(%rax), %ymm6 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm23 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm2, %zmm4, %zmm23 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm2, %zmm4, %zmm23 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm2 = [65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535] ; AVX512F-FAST-NEXT: vpternlogq $184, %zmm1, %zmm2, %zmm23 ; AVX512F-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload @@ -11790,7 +11789,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastd (%rax), %ymm5 ; AVX512F-FAST-NEXT: vpbroadcastd 4(%rax), %ymm6 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm20 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm1, %zmm4, %zmm20 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm1, %zmm4, %zmm20 ; AVX512F-FAST-NEXT: vpternlogq $184, %zmm9, %zmm2, %zmm20 ; AVX512F-FAST-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm4[10,11,8,9,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,u,u,u,u,26,27,24,25,u,u,u,u] @@ -11835,7 +11834,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vbroadcasti64x4 {{.*#+}} zmm5 = [0,13,4,0,0,14,5,0,0,13,4,0,0,14,5,0] ; AVX512F-FAST-NEXT: # zmm5 = mem[0,1,2,3,0,1,2,3] ; AVX512F-FAST-NEXT: vpermd %zmm1, %zmm5, %zmm19 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm2, %zmm25, %zmm19 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm2, %zmm25, %zmm19 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm19 ; AVX512F-FAST-NEXT: vmovdqa 32(%rcx), %xmm12 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdx), %xmm8 @@ -11851,7 +11850,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastd 96(%rax), %ymm2 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm21 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm30 = [65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm13, %zmm30, %zmm21 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm13, %zmm30, %zmm21 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm21 ; AVX512F-FAST-NEXT: vmovdqa64 %xmm24, %xmm2 ; AVX512F-FAST-NEXT: vmovdqa64 %xmm26, %xmm1 @@ -11904,7 +11903,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastd 104(%rax), %ymm6 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm2, %zmm2 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm22 = [65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm3, %zmm22, %zmm2 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm3, %zmm22, %zmm2 ; AVX512F-FAST-NEXT: vmovdqa64 %ymm18, %ymm3 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,28,29,26,27,28,29,26,27,28,29,30,31,30,31] ; AVX512F-FAST-NEXT: vpshufd {{.*#+}} ymm6 = ymm16[3,3,3,3,7,7,7,7] @@ -11918,7 +11917,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpermd %ymm14, %ymm31, %ymm9 ; AVX512F-FAST-NEXT: vpbroadcastd 32(%rax), %ymm18 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm18, %zmm9, %zmm9 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm10, %zmm30, %zmm9 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm10, %zmm30, %zmm9 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm9 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm1 = [65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0,0,65535,65535,65535,65535,0,0] ; AVX512F-FAST-NEXT: vpternlogq $184, %zmm0, %zmm1, %zmm2 @@ -11944,7 +11943,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vpbroadcastd 36(%rax), %ymm3 ; AVX512F-FAST-NEXT: vpbroadcastd 40(%rax), %ymm5 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm5, %zmm3, %zmm3 -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm4, %zmm22, %zmm3 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm4, %zmm22, %zmm3 ; AVX512F-FAST-NEXT: vpternlogq $184, %zmm0, %zmm1, %zmm3 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = [65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535,65535,65535,65535,65535,65535,0,65535] ; AVX512F-FAST-NEXT: vpternlogq $184, %ymm7, %ymm0, %ymm29 @@ -11955,7 +11954,7 @@ define void @store_i16_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm26, %zmm0, %zmm1 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm29[0,1,2,3],zmm1[4,5,6,7] ; AVX512F-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm12 # 64-byte Reload -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm12 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm12 ; AVX512F-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload ; AVX512F-FAST-NEXT: vpunpckhwd (%rsp), %xmm1, %xmm1 # 16-byte Folded Reload ; AVX512F-FAST-NEXT: # xmm1 = xmm1[4],mem[4],xmm1[5],mem[5],xmm1[6],mem[6],xmm1[7],mem[7] diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll index 8ccb4ef56d9c..f727622682cf 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-3.ll @@ -567,8 +567,8 @@ define void @store_i8_stride3_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm3 ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] ; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3] -; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm2 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] -; AVX512BW-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3] +; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm2 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] +; AVX512BW-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3] ; AVX512BW-NEXT: vpshufb %ymm2, %ymm0, %ymm0 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 ; AVX512BW-NEXT: vpshufb %zmm2, %zmm1, %zmm1 @@ -1086,8 +1086,8 @@ define void @store_i8_stride3_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] ; AVX512BW-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3] ; AVX512BW-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm2 -; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] -; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3] +; AVX512BW-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] +; AVX512BW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3] ; AVX512BW-NEXT: vpshufb %zmm3, %zmm2, %zmm2 ; AVX512BW-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm4 ; AVX512BW-NEXT: vpshufb %zmm3, %zmm4, %zmm4 diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll index f4fda97c0817..4174fde81747 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-5.ll @@ -927,7 +927,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vporq %zmm7, %zmm5, %zmm5 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm6 = ; AVX512F-SLOW-NEXT: vpermd %zmm1, %zmm6, %zmm6 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 ; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u],zero,zero,xmm3[10,11,u],zero,zero,xmm3[12,13,u],zero,zero,xmm3[14,15,u] ; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] @@ -966,7 +966,7 @@ define void @store_i8_stride5_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm6 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm7 = ; AVX512F-FAST-NEXT: vpermd %zmm6, %zmm7, %zmm6 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 ; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u],zero,zero,xmm3[10,11,u],zero,zero,xmm3[12,13,u],zero,zero,xmm3[14,15,u] ; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] @@ -1886,7 +1886,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vmovdqa (%r8), %xmm6 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm8 = ; AVX512F-SLOW-NEXT: vpermd %zmm6, %zmm8, %zmm6 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm6 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm5 = ymm4[u,u,u],zero,ymm4[13,u,u,u],zero,ymm4[14,u,u,u],zero,ymm4[15,u,u,u],zero,ymm4[16,u,u,u],zero,ymm4[17,u,u,u],zero,ymm4[18,u,u] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm8 = ymm3[u,u,u,13],zero,ymm3[u,u,u,14],zero,ymm3[u,u,u,15],zero,ymm3[u,u,u,16],zero,ymm3[u,u,u,17],zero,ymm3[u,u,u,18],zero,ymm3[u,u] ; AVX512F-SLOW-NEXT: vpor %ymm5, %ymm8, %ymm5 @@ -1968,7 +1968,7 @@ define void @store_i8_stride5_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = mem[0,1,2,3,0,1,2,3] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm7 = ; AVX512F-FAST-NEXT: vpermd %zmm4, %zmm7, %zmm7 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm7 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm7 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm5 = ymm3[u,u,u],zero,ymm3[13,u,u,u],zero,ymm3[14,u,u,u],zero,ymm3[15,u,u,u],zero,ymm3[16,u,u,u],zero,ymm3[17,u,u,u],zero,ymm3[18,u,u] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm8 = ymm2[u,u,u,13],zero,ymm2[u,u,u,14],zero,ymm2[u,u,u,15],zero,ymm2[u,u,u,16],zero,ymm2[u,u,u,17],zero,ymm2[u,u,u,18],zero,ymm2[u,u] ; AVX512F-FAST-NEXT: vpor %ymm5, %ymm8, %ymm5 @@ -3890,36 +3890,37 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm15 = [128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128,128,128] ; AVX512F-SLOW-NEXT: vpshufb %ymm15, %ymm3, %ymm0 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %ymm2 -; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm7 = <12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19> -; AVX512F-SLOW-NEXT: vpshufb %ymm7, %ymm2, %ymm1 +; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm9 = <12,13,128,15,12,13,14,128,12,13,14,15,128,u,u,u,16,128,18,19,16,17,128,19,16,17,18,128,16,17,18,19> +; AVX512F-SLOW-NEXT: vpshufb %ymm9, %ymm2, %ymm1 ; AVX512F-SLOW-NEXT: vpor %ymm0, %ymm1, %ymm0 ; AVX512F-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm8 = <8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11> -; AVX512F-SLOW-NEXT: vpshufb %xmm8, %xmm1, %xmm0 -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm1, %xmm30 +; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm5 = <8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11> +; AVX512F-SLOW-NEXT: vpshufb %xmm5, %xmm1, %xmm0 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm1, %xmm16 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rsi), %xmm4 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm14 = <128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128> ; AVX512F-SLOW-NEXT: vpshufb %xmm14, %xmm4, %xmm1 ; AVX512F-SLOW-NEXT: vmovdqa64 %xmm4, %xmm31 ; AVX512F-SLOW-NEXT: vpor %xmm0, %xmm1, %xmm0 ; AVX512F-SLOW-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX512F-SLOW-NEXT: vmovdqa 32(%rcx), %ymm9 +; AVX512F-SLOW-NEXT: vmovdqa 32(%rcx), %ymm8 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm0 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128] -; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm9, %ymm4 +; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm8, %ymm4 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdx), %ymm11 ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm1 = ; AVX512F-SLOW-NEXT: vpshufb %ymm1, %ymm11, %ymm10 -; AVX512F-SLOW-NEXT: vporq %ymm4, %ymm10, %ymm18 +; AVX512F-SLOW-NEXT: vpor %ymm4, %ymm10, %ymm4 +; AVX512F-SLOW-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX512F-SLOW-NEXT: vmovdqa 32(%rcx), %xmm13 -; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm5 = <128,6,128,8,u,128,7,128,9,128,11,u,128,10,128,12> -; AVX512F-SLOW-NEXT: vpshufb %xmm5, %xmm13, %xmm4 -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm5, %xmm25 +; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm6 = <128,6,128,8,u,128,7,128,9,128,11,u,128,10,128,12> +; AVX512F-SLOW-NEXT: vpshufb %xmm6, %xmm13, %xmm4 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm6, %xmm25 ; AVX512F-SLOW-NEXT: vmovdqa 32(%rdx), %xmm10 -; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm5 = <6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128> -; AVX512F-SLOW-NEXT: vpshufb %xmm5, %xmm10, %xmm12 -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm5, %xmm26 -; AVX512F-SLOW-NEXT: vporq %xmm4, %xmm12, %xmm19 +; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm6 = <6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128> +; AVX512F-SLOW-NEXT: vpshufb %xmm6, %xmm10, %xmm12 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm6, %xmm26 +; AVX512F-SLOW-NEXT: vporq %xmm4, %xmm12, %xmm20 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm4 = ymm3[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = ymm3[3,u,5,u,u,4,u,6,u,8,u,u,7,u,9,u,19,u,21,u,u,20,u,22,u,24,u,u,23,u,25,u] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm22 @@ -3927,31 +3928,31 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm2[21],zero,zero,ymm2[20],zero,ymm2[22],zero,ymm2[24],zero,zero,ymm2[23],zero,ymm2[25],zero,zero ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm2, %zmm23 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm11[27],zero,zero,ymm11[26],zero,ymm11[28],zero,ymm11[30],zero,zero,ymm11[29],zero,ymm11[31],zero,zero -; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm9[19],zero,ymm9[21],zero,zero,ymm9[20],zero,ymm9[22],zero,ymm9[24],zero,zero,ymm9[23],zero +; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm8[19],zero,ymm8[21],zero,zero,ymm8[20],zero,ymm8[22],zero,ymm8[24],zero,zero,ymm8[23],zero ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm24 ; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %ymm12 ; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm12, %ymm0 ; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %ymm6 ; AVX512F-SLOW-NEXT: vpshufb %ymm1, %ymm6, %ymm1 -; AVX512F-SLOW-NEXT: vporq %ymm0, %ymm1, %ymm20 -; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %ymm5 -; AVX512F-SLOW-NEXT: vpshufb %ymm15, %ymm5, %ymm2 +; AVX512F-SLOW-NEXT: vporq %ymm0, %ymm1, %ymm19 +; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %ymm7 +; AVX512F-SLOW-NEXT: vpshufb %ymm15, %ymm7, %ymm2 ; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm4 -; AVX512F-SLOW-NEXT: vpshufb %ymm7, %ymm4, %ymm3 +; AVX512F-SLOW-NEXT: vpshufb %ymm9, %ymm4, %ymm3 ; AVX512F-SLOW-NEXT: vporq %ymm2, %ymm3, %ymm21 ; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %xmm0 -; AVX512F-SLOW-NEXT: vpshufb %xmm8, %xmm0, %xmm2 -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm0, %xmm16 -; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %xmm7 -; AVX512F-SLOW-NEXT: vpshufb %xmm14, %xmm7, %xmm3 -; AVX512F-SLOW-NEXT: vporq %xmm2, %xmm3, %xmm27 +; AVX512F-SLOW-NEXT: vpshufb %xmm5, %xmm0, %xmm3 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm0, %xmm17 +; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %xmm5 +; AVX512F-SLOW-NEXT: vpshufb %xmm14, %xmm5, %xmm9 +; AVX512F-SLOW-NEXT: vporq %xmm3, %xmm9, %xmm27 ; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %xmm1 ; AVX512F-SLOW-NEXT: vmovdqa64 %xmm25, %xmm0 ; AVX512F-SLOW-NEXT: vpshufb %xmm0, %xmm1, %xmm0 -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm1, %xmm17 -; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %xmm8 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm1, %xmm18 +; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %xmm9 ; AVX512F-SLOW-NEXT: vmovdqa64 %xmm26, %xmm1 -; AVX512F-SLOW-NEXT: vpshufb %xmm1, %xmm8, %xmm15 +; AVX512F-SLOW-NEXT: vpshufb %xmm1, %xmm9, %xmm15 ; AVX512F-SLOW-NEXT: vporq %xmm0, %xmm15, %xmm29 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = mem[1,1,2,2] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,1,1] @@ -3965,10 +3966,11 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb %ymm1, %ymm0, %ymm1 ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,2,1,1,4,6,5,5] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,3,2] -; AVX512F-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 +; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm30 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0] +; AVX512F-SLOW-NEXT: vpandnq %ymm0, %ymm30, %ymm0 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm25 ; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm0 = [9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12] -; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm9, %ymm9 +; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm8, %ymm8 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm14 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm12[19],zero,ymm12[21],zero,zero,ymm12[20],zero,ymm12[22],zero,ymm12[24],zero,zero,ymm12[23],zero ; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm12, %ymm12 ; AVX512F-SLOW-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25] @@ -3977,24 +3979,25 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb %ymm0, %ymm6, %ymm2 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm6[27],zero,zero,ymm6[26],zero,ymm6[28],zero,ymm6[30],zero,zero,ymm6[29],zero,ymm6[31],zero,zero ; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm10 = xmm13[0],xmm10[0],xmm13[1],xmm10[1],xmm13[2],xmm10[2],xmm13[3],xmm10[3],xmm13[4],xmm10[4],xmm13[5],xmm10[5],xmm13[6],xmm10[6],xmm13[7],xmm10[7] -; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm13 = ymm5[3,u,5,u,u,4,u,6,u,8,u,u,7,u,9,u,19,u,21,u,u,20,u,22,u,24,u,u,23,u,25,u] -; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm5 = ymm5[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30] +; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm13 = ymm7[3,u,5,u,u,4,u,6,u,8,u,u,7,u,9,u,19,u,21,u,u,20,u,22,u,24,u,u,23,u,25,u] +; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm7 = ymm7[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm4[21],zero,zero,ymm4[20],zero,ymm4[22],zero,ymm4[24],zero,zero,ymm4[23],zero,ymm4[25],zero,zero ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm3 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm4[26],zero,ymm4[28],zero,zero,ymm4[27],zero,ymm4[29],zero,ymm4[31],zero,zero,ymm4[30],zero -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm30, %xmm1 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm16, %xmm1 ; AVX512F-SLOW-NEXT: vmovdqa64 %xmm31, %xmm4 ; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm4 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3],xmm1[4],xmm4[4],xmm1[5],xmm4[5],xmm1[6],xmm4[6],xmm1[7],xmm4[7] ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm1 = <4,u,5,5,5,5,u,6,6,6,6,u,7,7,7,7> -; AVX512F-SLOW-NEXT: vpermd %zmm15, %zmm1, %zmm30 -; AVX512F-SLOW-NEXT: vmovdqa64 (%r8), %zmm31 +; AVX512F-SLOW-NEXT: vpermd %zmm15, %zmm1, %zmm31 +; AVX512F-SLOW-NEXT: vmovdqa64 (%r8), %zmm16 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm1 = <6,6,6,u,7,7,7,7,u,16,16,16,16,u,17,17> -; AVX512F-SLOW-NEXT: vpermi2d %zmm15, %zmm31, %zmm1 -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm16, %xmm15 -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm15[0],xmm7[0],xmm15[1],xmm7[1],xmm15[2],xmm7[2],xmm15[3],xmm7[3],xmm15[4],xmm7[4],xmm15[5],xmm7[5],xmm15[6],xmm7[6],xmm15[7],xmm7[7] +; AVX512F-SLOW-NEXT: vpermi2d %zmm15, %zmm16, %zmm1 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm17, %xmm15 +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm15[0],xmm5[0],xmm15[1],xmm5[1],xmm15[2],xmm5[2],xmm15[3],xmm5[3],xmm15[4],xmm5[4],xmm15[5],xmm5[5],xmm15[6],xmm5[6],xmm15[7],xmm5[7] ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm15 = <0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13> ; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm4, %xmm4 -; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm7, %xmm7 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm9 = ymm9[2,2,3,3] +; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm5, %xmm5 +; AVX512F-SLOW-NEXT: vinserti32x4 $2, %xmm27, %zmm5, %zmm5 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm11 = ymm11[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,3,3] @@ -4002,65 +4005,64 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,2,3,3] -; AVX512F-SLOW-NEXT: vinserti32x4 $2, %xmm27, %zmm7, %zmm27 +; AVX512F-SLOW-NEXT: vmovdqa64 %xmm18, %xmm15 +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm9 = xmm15[0],xmm9[0],xmm15[1],xmm9[1],xmm15[2],xmm9[2],xmm15[3],xmm9[3],xmm15[4],xmm9[4],xmm15[5],xmm9[5],xmm15[6],xmm9[6],xmm15[7],xmm9[7] ; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} xmm15 = <2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8> ; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm10, %xmm10 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm10 = ymm10[0,0,1,1] -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,2,3,3] +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,0,1,1] -; AVX512F-SLOW-NEXT: vmovdqa64 %xmm17, %xmm7 -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm8 = xmm7[0],xmm8[0],xmm7[1],xmm8[1],xmm7[2],xmm8[2],xmm7[3],xmm8[3],xmm7[4],xmm8[4],xmm7[5],xmm8[5],xmm7[6],xmm8[6],xmm7[7],xmm8[7] -; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm8, %xmm8 -; AVX512F-SLOW-NEXT: vinserti32x4 $2, %xmm29, %zmm8, %zmm8 +; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm9, %xmm9 +; AVX512F-SLOW-NEXT: vinserti32x4 $2, %xmm29, %zmm9, %zmm9 ; AVX512F-SLOW-NEXT: vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm15 # 32-byte Folded Reload ; AVX512F-SLOW-NEXT: # ymm15 = mem[0,0,1,1] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm15, %zmm15 # 32-byte Folded Reload -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm16 = ymm19[0,0,1,1] -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm18, %zmm16, %zmm16 -; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm17 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0] -; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm15, %zmm17, %zmm16 -; AVX512F-SLOW-NEXT: vpternlogq $248, %zmm28, %zmm16, %zmm26 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm17 = ymm20[0,0,1,1] +; AVX512F-SLOW-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm17, %zmm17 # 32-byte Folded Reload +; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm18 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0] +; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm15, %zmm18, %zmm17 +; AVX512F-SLOW-NEXT: vpternlogq $248, %zmm28, %zmm17, %zmm26 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm15 = zmm22[2,2,3,3,6,6,7,7] +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm17 = zmm23[2,2,3,3,6,6,7,7] +; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm20 = [18374966859431608575,18374966859431608575,18446463693966278400,18446463693966278400,72056498804555775,72056498804555775,18374967950370078975,18374967950370078975] +; AVX512F-SLOW-NEXT: vpternlogq $248, %zmm20, %zmm15, %zmm17 +; AVX512F-SLOW-NEXT: vpandq %ymm20, %ymm8, %ymm8 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm8, %zmm11, %zmm8 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm11 = zmm24[2,2,3,3,6,6,7,7] +; AVX512F-SLOW-NEXT: vporq %zmm11, %zmm8, %zmm8 +; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm11 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255] +; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm17, %zmm11, %zmm8 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm8, %zmm30, %zmm31 ; AVX512F-SLOW-NEXT: vpor %ymm2, %ymm14, %ymm2 -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm20, %zmm2 -; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm14 = [18374966859431608575,18374966859431608575,18446463693966278400,18446463693966278400,72056498804555775,72056498804555775,18374967950370078975,18374967950370078975] -; AVX512F-SLOW-NEXT: vpternlogq $248, %ymm14, %ymm13, %ymm0 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm19, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogq $248, %ymm20, %ymm13, %ymm0 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm21, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm2, %zmm17, %zmm0 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm2 = zmm22[2,2,3,3,6,6,7,7] -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm13 = zmm23[2,2,3,3,6,6,7,7] -; AVX512F-SLOW-NEXT: vpternlogq $248, %zmm14, %zmm2, %zmm13 -; AVX512F-SLOW-NEXT: vpternlogq $248, %ymm14, %ymm12, %ymm6 -; AVX512F-SLOW-NEXT: vpand %ymm14, %ymm9, %ymm2 -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm11, %zmm2 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm9 = zmm24[2,2,3,3,6,6,7,7] -; AVX512F-SLOW-NEXT: vporq %zmm9, %zmm2, %zmm2 -; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm9 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255] -; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm13, %zmm9, %zmm2 -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm6, %zmm6 -; AVX512F-SLOW-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm5, %ymm3 +; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm2, %zmm18, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogq $248, %ymm20, %ymm12, %ymm6 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm6, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm3 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm3 -; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm6, %zmm9, %zmm3 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm30 +; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm2, %zmm11, %zmm3 ; AVX512F-SLOW-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm25 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm1 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm0 = zmm27[0,0,1,1,4,4,5,5] -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm2 = zmm8[0,0,1,1,4,4,5,5] +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm1 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm0 = zmm5[0,0,1,1,4,4,5,5] +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm2 = zmm9[0,0,1,1,4,4,5,5] ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm0 = -; AVX512F-SLOW-NEXT: vpermd %zmm31, %zmm0, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 +; AVX512F-SLOW-NEXT: vpermd %zmm16, %zmm0, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm25, 64(%r9) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, (%r9) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm1, 128(%r9) -; AVX512F-SLOW-NEXT: vmovdqa64 %zmm30, 256(%r9) +; AVX512F-SLOW-NEXT: vmovdqa64 %zmm31, 256(%r9) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm26, 192(%r9) ; AVX512F-SLOW-NEXT: vzeroupper ; AVX512F-SLOW-NEXT: retq ; ; AVX512F-FAST-LABEL: store_i8_stride5_vf64: ; AVX512F-FAST: # %bb.0: -; AVX512F-FAST-NEXT: pushq %rax +; AVX512F-FAST-NEXT: subq $24, %rsp ; AVX512F-FAST-NEXT: vmovdqa 32(%rsi), %ymm3 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm8 = [128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128,128,128] ; AVX512F-FAST-NEXT: vpshufb %ymm8, %ymm3, %ymm0 @@ -4069,14 +4071,14 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb %ymm11, %ymm1, %ymm2 ; AVX512F-FAST-NEXT: vpor %ymm0, %ymm2, %ymm0 ; AVX512F-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %xmm2 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm6 = <8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11> -; AVX512F-FAST-NEXT: vpshufb %xmm6, %xmm2, %xmm0 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm2, %xmm19 +; AVX512F-FAST-NEXT: vmovdqa 32(%rdi), %xmm0 +; AVX512F-FAST-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm5 = <8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11> +; AVX512F-FAST-NEXT: vpshufb %xmm5, %xmm0, %xmm0 ; AVX512F-FAST-NEXT: vmovdqa 32(%rsi), %xmm4 ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm13 = <128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128> ; AVX512F-FAST-NEXT: vpshufb %xmm13, %xmm4, %xmm2 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm4, %xmm18 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm4, %xmm19 ; AVX512F-FAST-NEXT: vpor %xmm0, %xmm2, %xmm0 ; AVX512F-FAST-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX512F-FAST-NEXT: vmovdqa 32(%rcx), %ymm7 @@ -4092,9 +4094,9 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb %xmm4, %xmm12, %xmm2 ; AVX512F-FAST-NEXT: vmovdqa64 %xmm4, %xmm25 ; AVX512F-FAST-NEXT: vmovdqa 32(%rdx), %xmm10 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm5 = <6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128> -; AVX512F-FAST-NEXT: vpshufb %xmm5, %xmm10, %xmm4 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm5, %xmm26 +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm6 = <6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128> +; AVX512F-FAST-NEXT: vpshufb %xmm6, %xmm10, %xmm4 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm6, %xmm26 ; AVX512F-FAST-NEXT: vpor %xmm2, %xmm4, %xmm2 ; AVX512F-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm2 = ymm3[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30] @@ -4104,140 +4106,139 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm1[21],zero,zero,ymm1[20],zero,ymm1[22],zero,ymm1[24],zero,zero,ymm1[23],zero,ymm1[25],zero,zero ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm22 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm9[27],zero,zero,ymm9[26],zero,ymm9[28],zero,ymm9[30],zero,zero,ymm9[29],zero,ymm9[31],zero,zero -; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128,128,128,19,128,21,128,128,20,128,22,128,24,128,128,23,128] -; AVX512F-FAST-NEXT: # ymm3 = mem[0,1,0,1] -; AVX512F-FAST-NEXT: vpshufb %ymm3, %ymm7, %ymm2 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm3, %ymm31 +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm2 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm7[19],zero,ymm7[21],zero,zero,ymm7[20],zero,ymm7[22],zero,ymm7[24],zero,zero,ymm7[23],zero ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm23 -; AVX512F-FAST-NEXT: vmovdqa (%rcx), %ymm5 -; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm5, %ymm0 +; AVX512F-FAST-NEXT: vmovdqa (%rcx), %ymm4 +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm4, %ymm0 ; AVX512F-FAST-NEXT: vmovdqa (%rdx), %ymm14 ; AVX512F-FAST-NEXT: vpshufb %ymm15, %ymm14, %ymm1 ; AVX512F-FAST-NEXT: vporq %ymm0, %ymm1, %ymm24 ; AVX512F-FAST-NEXT: vmovdqa (%rsi), %ymm15 ; AVX512F-FAST-NEXT: vpshufb %ymm8, %ymm15, %ymm0 -; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm4 -; AVX512F-FAST-NEXT: vpshufb %ymm11, %ymm4, %ymm1 +; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm6 +; AVX512F-FAST-NEXT: vpshufb %ymm11, %ymm6, %ymm1 ; AVX512F-FAST-NEXT: vporq %ymm0, %ymm1, %ymm20 -; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm1 -; AVX512F-FAST-NEXT: vpshufb %xmm6, %xmm1, %xmm0 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm1, %xmm16 +; AVX512F-FAST-NEXT: vmovdqa (%rdi), %xmm0 +; AVX512F-FAST-NEXT: vpshufb %xmm5, %xmm0, %xmm1 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm0, %xmm16 ; AVX512F-FAST-NEXT: vmovdqa (%rsi), %xmm3 ; AVX512F-FAST-NEXT: vpshufb %xmm13, %xmm3, %xmm2 -; AVX512F-FAST-NEXT: vporq %xmm0, %xmm2, %xmm28 +; AVX512F-FAST-NEXT: vporq %xmm1, %xmm2, %xmm28 ; AVX512F-FAST-NEXT: vmovdqa (%rcx), %xmm1 ; AVX512F-FAST-NEXT: vmovdqa64 %xmm25, %xmm0 -; AVX512F-FAST-NEXT: vpshufb %xmm0, %xmm1, %xmm0 +; AVX512F-FAST-NEXT: vpshufb %xmm0, %xmm1, %xmm8 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm1, %xmm18 +; AVX512F-FAST-NEXT: vmovdqa (%rdx), %xmm1 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm26, %xmm0 +; AVX512F-FAST-NEXT: vpshufb %xmm0, %xmm1, %xmm11 ; AVX512F-FAST-NEXT: vmovdqa64 %xmm1, %xmm17 -; AVX512F-FAST-NEXT: vmovdqa (%rdx), %xmm11 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm26, %xmm1 -; AVX512F-FAST-NEXT: vpshufb %xmm1, %xmm11, %xmm8 -; AVX512F-FAST-NEXT: vporq %xmm0, %xmm8, %xmm29 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm0 = [1,1,2,2,2,2,2,2] +; AVX512F-FAST-NEXT: vporq %xmm8, %xmm11, %xmm29 +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm11 = [1,1,2,2,2,2,2,2] ; AVX512F-FAST-NEXT: vmovdqa 32(%r8), %ymm8 -; AVX512F-FAST-NEXT: vpermd %ymm8, %ymm0, %ymm0 +; AVX512F-FAST-NEXT: vpermd %ymm8, %ymm11, %ymm11 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm25 = [255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255] -; AVX512F-FAST-NEXT: vpandnq %ymm0, %ymm25, %ymm0 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm1 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128] -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm8, %ymm13 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm13, %zmm0, %zmm26 -; AVX512F-FAST-NEXT: vmovdqa (%r8), %ymm2 -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm2, %ymm1 -; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm30 = <4,u,5,5,5,5,u,6,30,30,30,u,31,31,31,31> -; AVX512F-FAST-NEXT: vpermd %ymm2, %ymm30, %ymm27 -; AVX512F-FAST-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm27, %ymm27 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm27, %zmm1, %zmm27 -; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm1 = [9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12] -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm7, %ymm7 -; AVX512F-FAST-NEXT: vmovdqa64 %ymm31, %ymm0 -; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm5, %ymm13 -; AVX512F-FAST-NEXT: vpshufb %ymm1, %ymm5, %ymm6 -; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25] -; AVX512F-FAST-NEXT: # ymm5 = mem[0,1,0,1] -; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm9, %ymm9 -; AVX512F-FAST-NEXT: vpshufb %ymm5, %ymm14, %ymm5 +; AVX512F-FAST-NEXT: vpandnq %ymm11, %ymm25, %ymm11 +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm0 = [12,128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128] +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm8, %ymm13 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm13, %zmm11, %zmm26 +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm31 = <4,u,5,5,5,5,u,6,30,30,30,u,31,31,31,31> +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm30 = [255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0,255,255,255,255,0] +; AVX512F-FAST-NEXT: vmovdqa (%r8), %ymm11 +; AVX512F-FAST-NEXT: vpermd %ymm11, %ymm31, %ymm27 +; AVX512F-FAST-NEXT: vpandnq %ymm27, %ymm30, %ymm27 +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm11, %ymm0 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm27, %zmm0, %zmm27 +; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm0 = [9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12,9,14,11,0,13,10,15,12] +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm7, %ymm7 +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm13 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm4[19],zero,ymm4[21],zero,zero,ymm4[20],zero,ymm4[22],zero,ymm4[24],zero,zero,ymm4[23],zero +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm4, %ymm5 +; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25,18,19,128,21,128,21,20,128,22,128,24,128,22,23,128,25] +; AVX512F-FAST-NEXT: # ymm0 = mem[0,1,0,1] +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm9, %ymm9 +; AVX512F-FAST-NEXT: vpshufb %ymm0, %ymm14, %ymm4 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm14 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm14[27],zero,zero,ymm14[26],zero,ymm14[28],zero,ymm14[30],zero,zero,ymm14[29],zero,ymm14[31],zero,zero ; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} xmm10 = xmm12[0],xmm10[0],xmm12[1],xmm10[1],xmm12[2],xmm10[2],xmm12[3],xmm10[3],xmm12[4],xmm10[4],xmm12[5],xmm10[5],xmm12[6],xmm10[6],xmm12[7],xmm10[7] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm12 = ymm15[3,u,5,u,u,4,u,6,u,8,u,u,7,u,9,u,19,u,21,u,u,20,u,22,u,24,u,u,23,u,25,u] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm15 = ymm15[11,u,u,10,u,12,u,u,u,u,13,u,15,u,u,14,27,u,u,26,u,28,u,u,u,u,29,u,31,u,u,30] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm31 = ymm7[2,2,3,3] -; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm7 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm4[21],zero,zero,ymm4[20],zero,ymm4[22],zero,ymm4[24],zero,zero,ymm4[23],zero,ymm4[25],zero,zero -; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm4[26],zero,ymm4[28],zero,zero,ymm4[27],zero,ymm4[29],zero,ymm4[31],zero,zero,ymm4[30],zero -; AVX512F-FAST-NEXT: vmovdqa64 %xmm19, %xmm0 -; AVX512F-FAST-NEXT: vmovdqa64 %xmm18, %xmm1 +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm2 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm6[21],zero,zero,ymm6[20],zero,ymm6[22],zero,ymm6[24],zero,zero,ymm6[23],zero,ymm6[25],zero,zero +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm6 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm6[26],zero,ymm6[28],zero,zero,ymm6[27],zero,ymm6[29],zero,ymm6[31],zero,zero,ymm6[30],zero +; AVX512F-FAST-NEXT: vmovdqa64 %xmm19, %xmm1 +; AVX512F-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload ; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] ; AVX512F-FAST-NEXT: vmovdqa64 %xmm16, %xmm1 ; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3],xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = <0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13> ; AVX512F-FAST-NEXT: vpshufb %xmm1, %xmm0, %xmm0 ; AVX512F-FAST-NEXT: vpshufb %xmm1, %xmm3, %xmm1 -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm3 = ymm9[2,2,3,3] +; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm28, %zmm1, %zmm28 +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm1 = ymm7[2,2,3,3] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm9[2,2,3,3] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm9 = ymm13[2,2,3,3] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm5 = ymm5[2,2,3,3] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,3,3] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,2,3,3] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,2,3,3] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm13 = ymm14[2,2,3,3] -; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm28, %zmm1, %zmm18 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm14 = <2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8> -; AVX512F-FAST-NEXT: vpshufb %xmm14, %xmm10, %xmm10 +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,3,3] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm13 = ymm5[2,2,3,3] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm14 = ymm14[2,2,3,3] +; AVX512F-FAST-NEXT: vmovdqa64 %xmm18, %xmm3 +; AVX512F-FAST-NEXT: vmovdqa64 %xmm17, %xmm5 +; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} xmm5 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3],xmm3[4],xmm5[4],xmm3[5],xmm5[5],xmm3[6],xmm5[6],xmm3[7],xmm5[7] +; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} xmm3 = <2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8> +; AVX512F-FAST-NEXT: vpshufb %xmm3, %xmm10, %xmm10 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm10 = ymm10[0,0,1,1] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,3,3] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm4 = ymm4[2,2,3,3] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,2,3,3] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,0,1,1] -; AVX512F-FAST-NEXT: vmovdqa64 %xmm17, %xmm1 -; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} xmm11 = xmm1[0],xmm11[0],xmm1[1],xmm11[1],xmm1[2],xmm11[2],xmm1[3],xmm11[3],xmm1[4],xmm11[4],xmm1[5],xmm11[5],xmm1[6],xmm11[6],xmm1[7],xmm11[7] -; AVX512F-FAST-NEXT: vpshufb %xmm14, %xmm11, %xmm11 -; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm29, %zmm11, %zmm11 -; AVX512F-FAST-NEXT: vmovdqa64 (%r8), %zmm14 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm14, %zmm2 +; AVX512F-FAST-NEXT: vpshufb %xmm3, %xmm5, %xmm3 +; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm29, %zmm3, %zmm3 +; AVX512F-FAST-NEXT: vmovdqa64 (%r8), %zmm5 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm5, %zmm11 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm21 = zmm21[2,2,3,3,6,6,7,7] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm22 = zmm22[2,2,3,3,6,6,7,7] -; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm28 = [18374966859431608575,18374966859431608575,18446463693966278400,18446463693966278400,72056498804555775,72056498804555775,18374967950370078975,18374967950370078975] -; AVX512F-FAST-NEXT: vpternlogq $248, %zmm28, %zmm21, %zmm22 -; AVX512F-FAST-NEXT: vpandq %ymm28, %ymm31, %ymm21 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm21, %zmm3, %zmm3 -; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm21 = zmm23[2,2,3,3,6,6,7,7] -; AVX512F-FAST-NEXT: vporq %zmm21, %zmm3, %zmm3 -; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm21 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255] -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm22, %zmm21, %zmm3 -; AVX512F-FAST-NEXT: vpermt2d %zmm14, %zmm30, %zmm8 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm8 +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm29 = [18374966859431608575,18374966859431608575,18446463693966278400,18446463693966278400,72056498804555775,72056498804555775,18374967950370078975,18374967950370078975] +; AVX512F-FAST-NEXT: vpternlogq $248, %zmm29, %zmm21, %zmm22 +; AVX512F-FAST-NEXT: vpandq %ymm29, %ymm1, %ymm1 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm7, %zmm1 +; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm7 = zmm23[2,2,3,3,6,6,7,7] +; AVX512F-FAST-NEXT: vporq %zmm7, %zmm1, %zmm1 +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm7 = [0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255] +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm22, %zmm7, %zmm1 +; AVX512F-FAST-NEXT: vpermt2d %zmm5, %zmm31, %zmm8 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm1, %zmm30, %zmm8 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm8, 256(%r9) -; AVX512F-FAST-NEXT: vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Folded Reload -; AVX512F-FAST-NEXT: # ymm3 = mem[0,0,1,1] -; AVX512F-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm3, %zmm3 # 32-byte Folded Reload +; AVX512F-FAST-NEXT: vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Folded Reload +; AVX512F-FAST-NEXT: # ymm1 = mem[0,0,1,1] +; AVX512F-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm1, %zmm1 # 32-byte Folded Reload ; AVX512F-FAST-NEXT: vpermq $80, {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Folded Reload ; AVX512F-FAST-NEXT: # ymm8 = mem[0,0,1,1] ; AVX512F-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8, %zmm8 # 32-byte Folded Reload ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm16 = [255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0,0,255,255,255,0] -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm3, %zmm16, %zmm8 +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm1, %zmm16, %zmm8 ; AVX512F-FAST-NEXT: vpternlogq $248, %zmm25, %zmm8, %zmm26 -; AVX512F-FAST-NEXT: vpor %ymm5, %ymm9, %ymm3 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm3, %zmm24, %zmm3 -; AVX512F-FAST-NEXT: vpternlogq $248, %ymm28, %ymm12, %ymm7 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm20, %zmm5 -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm3, %zmm16, %zmm5 -; AVX512F-FAST-NEXT: vpternlogq $248, %ymm28, %ymm6, %ymm13 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm13, %zmm3 -; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm15, %ymm4 -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm4, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm3, %zmm21, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm27 -; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm3 = <6,6,6,u,7,7,7,7,u,8,8,8,8,u,9,9> -; AVX512F-FAST-NEXT: vpermd %zmm14, %zmm3, %zmm3 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm3 -; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm0 = zmm18[0,0,1,1,4,4,5,5] -; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm11[0,0,1,1,4,4,5,5] -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 +; AVX512F-FAST-NEXT: vpor %ymm4, %ymm9, %ymm1 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm24, %zmm1 +; AVX512F-FAST-NEXT: vpternlogq $248, %ymm29, %ymm12, %ymm2 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm20, %zmm2 +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm1, %zmm16, %zmm2 +; AVX512F-FAST-NEXT: vpternlogq $248, %ymm29, %ymm13, %ymm14 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm14, %zmm1 +; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm15, %ymm6 +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm6, %zmm0 +; AVX512F-FAST-NEXT: vpternlogq $226, %zmm1, %zmm7, %zmm0 +; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm27 +; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm1 = <6,6,6,u,7,7,7,7,u,8,8,8,8,u,9,9> +; AVX512F-FAST-NEXT: vpermd %zmm5, %zmm1, %zmm1 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 +; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm0 = zmm28[0,0,1,1,4,4,5,5] +; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm2 = zmm3[0,0,1,1,4,4,5,5] +; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2 ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = -; AVX512F-FAST-NEXT: vpermd %zmm2, %zmm0, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0 +; AVX512F-FAST-NEXT: vpermd %zmm11, %zmm0, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm27, 64(%r9) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm0, (%r9) -; AVX512F-FAST-NEXT: vmovdqa64 %zmm3, 128(%r9) +; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, 128(%r9) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm26, 192(%r9) -; AVX512F-FAST-NEXT: popq %rax +; AVX512F-FAST-NEXT: addq $24, %rsp ; AVX512F-FAST-NEXT: vzeroupper ; AVX512F-FAST-NEXT: retq ; @@ -4424,37 +4425,37 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-LABEL: store_i8_stride5_vf64: ; AVX512BW-FAST: # %bb.0: ; AVX512BW-FAST-NEXT: vmovdqa64 (%r8), %zmm5 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %ymm21 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %ymm20 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} ymm1 = [128,128,12,13,128,128,128,128,14,128,128,128,14,15,128,128,128,128,16,128,128,128,16,17,128,128,128,128,18,128,128,128] -; AVX512BW-FAST-NEXT: vpshufb %ymm1, %ymm21, %ymm0 +; AVX512BW-FAST-NEXT: vpshufb %ymm1, %ymm20, %ymm0 ; AVX512BW-FAST-NEXT: vmovdqa 32(%rcx), %ymm13 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} ymm2 = [128,128,128,128,13,128,128,128,128,14,128,128,128,128,15,128,128,128,128,16,128,128,128,128,17,128,128,128,128,18,128,128] ; AVX512BW-FAST-NEXT: vpshufb %ymm2, %ymm13, %ymm3 ; AVX512BW-FAST-NEXT: vpor %ymm0, %ymm3, %ymm0 ; AVX512BW-FAST-NEXT: vmovdqa (%rcx), %xmm6 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rcx), %xmm18 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rcx), %xmm17 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm8 = <128,6,128,8,u,128,7,128,9,128,11,u,128,10,128,12> -; AVX512BW-FAST-NEXT: vpshufb %xmm8, %xmm18, %xmm3 +; AVX512BW-FAST-NEXT: vpshufb %xmm8, %xmm17, %xmm3 ; AVX512BW-FAST-NEXT: vmovdqa (%rdx), %xmm7 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %xmm20 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %xmm19 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm10 = <6,128,8,128,u,7,128,9,128,11,128,u,10,128,12,128> -; AVX512BW-FAST-NEXT: vpshufb %xmm10, %xmm20, %xmm4 +; AVX512BW-FAST-NEXT: vpshufb %xmm10, %xmm19, %xmm4 ; AVX512BW-FAST-NEXT: vpor %xmm3, %xmm4, %xmm3 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,0,1,1] ; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0 ; AVX512BW-FAST-NEXT: vmovdqa (%rdi), %xmm9 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %xmm17 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %xmm16 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm12 = <8,128,u,7,128,9,128,u,128,u,10,128,12,128,u,11> -; AVX512BW-FAST-NEXT: vpshufb %xmm12, %xmm17, %xmm3 +; AVX512BW-FAST-NEXT: vpshufb %xmm12, %xmm16, %xmm3 ; AVX512BW-FAST-NEXT: vmovdqa (%rsi), %xmm11 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rsi), %xmm19 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rsi), %xmm18 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm14 = <128,8,u,128,7,128,9,u,11,u,128,10,128,12,u,128> -; AVX512BW-FAST-NEXT: vpshufb %xmm14, %xmm19, %xmm4 +; AVX512BW-FAST-NEXT: vpshufb %xmm14, %xmm18, %xmm4 ; AVX512BW-FAST-NEXT: vpor %xmm3, %xmm4, %xmm3 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm15 = ymm3[0,0,1,1] -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %ymm16 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %ymm21 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} ymm3 = <3,3,3,u,4,4,4,4> -; AVX512BW-FAST-NEXT: vpermd %ymm16, %ymm3, %ymm22 +; AVX512BW-FAST-NEXT: vpermd %ymm21, %ymm3, %ymm22 ; AVX512BW-FAST-NEXT: vmovdqa64 32(%rsi), %ymm23 ; AVX512BW-FAST-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14,0,0,13,2,15,0,1,14] ; AVX512BW-FAST-NEXT: movl $138547332, %eax # imm = 0x8421084 @@ -4464,80 +4465,77 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-NEXT: movabsq $-8330787646191410408, %rax # imm = 0x8C6318C6318C6318 ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm15, %zmm0 {%k2} -; AVX512BW-FAST-NEXT: vmovdqa64 32(%r8), %ymm24 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%r8), %ymm22 ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm15 = <1,1,2,2,2,2,2,2,27,27,27,27,u,28,28,28> -; AVX512BW-FAST-NEXT: vpermi2d %zmm5, %zmm24, %zmm15 +; AVX512BW-FAST-NEXT: vpermi2d %zmm5, %zmm22, %zmm15 ; AVX512BW-FAST-NEXT: movabsq $4760450083537948804, %rax # imm = 0x4210842108421084 ; AVX512BW-FAST-NEXT: kmovq %rax, %k3 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm15, %zmm0 {%k3} -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm22 = zmm23[0,1,2,3],mem[4,5,6,7] +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm23 = zmm23[0,1,2,3],mem[4,5,6,7] ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm15 = -; AVX512BW-FAST-NEXT: vpshufb %zmm15, %zmm22, %zmm22 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm22 = zmm22[2,2,3,3,6,6,7,7] -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm23 = zmm16[0,1,2,3],mem[4,5,6,7] -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm16 = -; AVX512BW-FAST-NEXT: vpshufb %zmm16, %zmm23, %zmm23 +; AVX512BW-FAST-NEXT: vpshufb %zmm15, %zmm23, %zmm23 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm23 = zmm23[2,2,3,3,6,6,7,7] -; AVX512BW-FAST-NEXT: vporq %zmm22, %zmm23, %zmm23 -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm22 = zmm21[0,1,2,3],mem[4,5,6,7] -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm21 = -; AVX512BW-FAST-NEXT: vpshufb %zmm21, %zmm22, %zmm22 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm25 = zmm22[2,2,3,3,6,6,7,7] +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm21 = zmm21[0,1,2,3],mem[4,5,6,7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm21 = zmm21[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm21[21],zero,zero,zmm21[20],zero,zmm21[22],zero,zmm21[24],zero,zero,zmm21[23],zero,zmm21[25],zero,zero,zmm21[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm21[58],zero,zmm21[60],zero,zero,zmm21[59],zero,zmm21[61],zero,zmm21[63],zero,zero,zmm21[62],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm21 = zmm21[2,2,3,3,6,6,7,7] +; AVX512BW-FAST-NEXT: vporq %zmm23, %zmm21, %zmm21 +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm20[0,1,2,3],mem[4,5,6,7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm20 = zmm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19],zero,zmm20[21],zero,zmm20[21,20],zero,zmm20[22],zero,zmm20[24],zero,zmm20[22,23],zero,zmm20[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,57],zero,zmm20[59],zero,zero,zmm20[58],zero,zmm20[60],zero,zmm20[62],zero,zero,zmm20[61],zero,zmm20[63],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm20 = zmm20[2,2,3,3,6,6,7,7] ; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm13 = zmm13[0,1,2,3],mem[4,5,6,7] -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm22 = -; AVX512BW-FAST-NEXT: vpshufb %zmm22, %zmm13, %zmm13 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm13 = zmm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm13[19],zero,zmm13[21],zero,zero,zmm13[20],zero,zmm13[22],zero,zmm13[24],zero,zero,zmm13[23],zero,zmm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm13[59],zero,zero,zmm13[58],zero,zmm13[60],zero,zmm13[62],zero,zero,zmm13[61],zero,zmm13[63],zero,zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm13 = zmm13[2,2,3,3,6,6,7,7] -; AVX512BW-FAST-NEXT: vporq %zmm25, %zmm13, %zmm13 +; AVX512BW-FAST-NEXT: vporq %zmm20, %zmm13, %zmm13 ; AVX512BW-FAST-NEXT: movabsq $1785168781326730801, %rax # imm = 0x18C6318C6318C631 ; AVX512BW-FAST-NEXT: kmovq %rax, %k3 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm23, %zmm13 {%k3} -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm23 = [4,6,5,5,5,5,4,6,30,30,30,30,31,31,31,31] -; AVX512BW-FAST-NEXT: vpermi2d %zmm5, %zmm24, %zmm23 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm21, %zmm13 {%k3} +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm20 = [4,6,5,5,5,5,4,6,30,30,30,30,31,31,31,31] +; AVX512BW-FAST-NEXT: vpermi2d %zmm5, %zmm22, %zmm20 ; AVX512BW-FAST-NEXT: movabsq $-8925843906633654008, %rax # imm = 0x8421084210842108 ; AVX512BW-FAST-NEXT: kmovq %rax, %k4 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm23, %zmm13 {%k4} -; AVX512BW-FAST-NEXT: vmovdqa64 (%rcx), %ymm23 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm24 = ymm23[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm23[27],zero,zero,ymm23[26],zero,ymm23[28],zero,ymm23[30],zero,zero,ymm23[29],zero,ymm23[31],zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm25 = ymm24[2,2,3,3] -; AVX512BW-FAST-NEXT: vmovdqa64 (%rdx), %ymm24 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm26 = ymm24[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm24[27],zero,zero,ymm24[26],zero,ymm24[28],zero,ymm24[30],zero,zero,ymm24[29],zero,ymm24[31],zero,zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm26 = ymm26[2,2,3,3] -; AVX512BW-FAST-NEXT: vporq %ymm25, %ymm26, %ymm25 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm18 = xmm18[0],xmm20[0],xmm18[1],xmm20[1],xmm18[2],xmm20[2],xmm18[3],xmm20[3],xmm18[4],xmm20[4],xmm18[5],xmm20[5],xmm18[6],xmm20[6],xmm18[7],xmm20[7] -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm26 = <2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8> -; AVX512BW-FAST-NEXT: vpshufb %xmm26, %xmm18, %xmm18 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm18 = ymm18[0,0,1,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm18, %zmm25, %zmm25 -; AVX512BW-FAST-NEXT: vmovdqa64 (%rsi), %ymm18 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm20 = ymm18[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,ymm18[26],zero,ymm18[28],zero,zero,zero,zero,ymm18[29],zero,ymm18[31],zero,zero,ymm18[30] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm27 = ymm20[2,2,3,3] -; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %ymm20 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm28 = ymm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm20[26],zero,ymm20[28],zero,zero,ymm20[27],zero,ymm20[29],zero,ymm20[31],zero,zero,ymm20[30],zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm28 = ymm28[2,2,3,3] -; AVX512BW-FAST-NEXT: vporq %ymm27, %ymm28, %ymm27 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm20, %zmm13 {%k4} +; AVX512BW-FAST-NEXT: vmovdqa64 (%rcx), %ymm20 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm21 = ymm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,ymm20[27],zero,zero,ymm20[26],zero,ymm20[28],zero,ymm20[30],zero,zero,ymm20[29],zero,ymm20[31],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm21 = ymm21[2,2,3,3] +; AVX512BW-FAST-NEXT: vmovdqa64 (%rdx), %ymm22 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm23 = ymm22[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm22[27],zero,zero,ymm22[26],zero,ymm22[28],zero,ymm22[30],zero,zero,ymm22[29],zero,ymm22[31],zero,zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm23 = ymm23[2,2,3,3] +; AVX512BW-FAST-NEXT: vporq %ymm21, %ymm23, %ymm21 ; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm17 = xmm17[0],xmm19[0],xmm17[1],xmm19[1],xmm17[2],xmm19[2],xmm17[3],xmm19[3],xmm17[4],xmm19[4],xmm17[5],xmm19[5],xmm17[6],xmm19[6],xmm17[7],xmm19[7] -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm19 = <0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13> +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm19 = <2,u,1,0,5,4,u,3,u,7,6,11,10,u,9,8> ; AVX512BW-FAST-NEXT: vpshufb %xmm19, %xmm17, %xmm17 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm17 = ymm17[0,0,1,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm17, %zmm27, %zmm17 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm25, %zmm17 {%k3} -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm25 = [6,6,6,6,7,7,7,7,8,8,8,8,8,8,9,9] -; AVX512BW-FAST-NEXT: vpermd %zmm5, %zmm25, %zmm5 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm17, %zmm21, %zmm21 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rsi), %ymm17 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm23 = ymm17[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,ymm17[26],zero,ymm17[28],zero,zero,zero,zero,ymm17[29],zero,ymm17[31],zero,zero,ymm17[30] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm23 = ymm23[2,2,3,3] +; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %ymm24 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm25 = ymm24[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm24[26],zero,ymm24[28],zero,zero,ymm24[27],zero,ymm24[29],zero,ymm24[31],zero,zero,ymm24[30],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm25 = ymm25[2,2,3,3] +; AVX512BW-FAST-NEXT: vporq %ymm23, %ymm25, %ymm23 +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm16 = xmm16[0],xmm18[0],xmm16[1],xmm18[1],xmm16[2],xmm18[2],xmm16[3],xmm18[3],xmm16[4],xmm18[4],xmm16[5],xmm18[5],xmm16[6],xmm18[6],xmm16[7],xmm18[7] +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm18 = <0,1,4,5,u,2,3,6,7,10,11,u,8,9,12,13> +; AVX512BW-FAST-NEXT: vpshufb %xmm18, %xmm16, %xmm16 +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm16 = ymm16[0,0,1,1] +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm16, %zmm23, %zmm16 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm21, %zmm16 {%k3} +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm21 = [6,6,6,6,7,7,7,7,8,8,8,8,8,8,9,9] +; AVX512BW-FAST-NEXT: vpermd %zmm5, %zmm21, %zmm5 ; AVX512BW-FAST-NEXT: movabsq $2380225041768974402, %rax # imm = 0x2108421084210842 ; AVX512BW-FAST-NEXT: kmovq %rax, %k3 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm5, %zmm17 {%k3} +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm5, %zmm16 {%k3} ; AVX512BW-FAST-NEXT: vpshufb %xmm8, %xmm6, %xmm5 ; AVX512BW-FAST-NEXT: vpshufb %xmm10, %xmm7, %xmm8 ; AVX512BW-FAST-NEXT: vpor %xmm5, %xmm8, %xmm5 ; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1],xmm6[2],xmm7[2],xmm6[3],xmm7[3],xmm6[4],xmm7[4],xmm6[5],xmm7[5],xmm6[6],xmm7[6],xmm6[7],xmm7[7] -; AVX512BW-FAST-NEXT: vpshufb %xmm26, %xmm6, %xmm6 +; AVX512BW-FAST-NEXT: vpshufb %xmm19, %xmm6, %xmm6 ; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm5, %zmm6, %zmm5 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm5 = zmm5[0,0,1,1,4,4,5,5] ; AVX512BW-FAST-NEXT: vpshufb %xmm12, %xmm9, %xmm6 ; AVX512BW-FAST-NEXT: vpshufb %xmm14, %xmm11, %xmm7 ; AVX512BW-FAST-NEXT: vpor %xmm6, %xmm7, %xmm6 ; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm9[0],xmm11[0],xmm9[1],xmm11[1],xmm9[2],xmm11[2],xmm9[3],xmm11[3],xmm9[4],xmm11[4],xmm9[5],xmm11[5],xmm9[6],xmm11[6],xmm9[7],xmm11[7] -; AVX512BW-FAST-NEXT: vpshufb %xmm19, %xmm7, %xmm7 +; AVX512BW-FAST-NEXT: vpshufb %xmm18, %xmm7, %xmm7 ; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm6, %zmm7, %zmm6 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm6 = zmm6[0,0,1,1,4,4,5,5] ; AVX512BW-FAST-NEXT: movabsq $-4165393823095705204, %rax # imm = 0xC6318C6318C6318C @@ -4549,22 +4547,22 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-NEXT: movabsq $595056260442243600, %rax # imm = 0x842108421084210 ; AVX512BW-FAST-NEXT: kmovq %rax, %k3 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm7, %zmm6 {%k3} -; AVX512BW-FAST-NEXT: vpshufb %ymm21, %ymm24, %ymm7 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3] -; AVX512BW-FAST-NEXT: vpshufb %ymm22, %ymm23, %ymm8 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,2,3,3] -; AVX512BW-FAST-NEXT: vpor %ymm7, %ymm8, %ymm7 -; AVX512BW-FAST-NEXT: vpshufb %ymm1, %ymm24, %ymm1 -; AVX512BW-FAST-NEXT: vpshufb %ymm2, %ymm23, %ymm2 +; AVX512BW-FAST-NEXT: vpshufb %ymm1, %ymm22, %ymm1 +; AVX512BW-FAST-NEXT: vpshufb %ymm2, %ymm20, %ymm2 ; AVX512BW-FAST-NEXT: vpor %ymm1, %ymm2, %ymm1 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm1, %zmm1 -; AVX512BW-FAST-NEXT: vpshufb %ymm15, %ymm18, %ymm2 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm22[18,19],zero,ymm22[21],zero,ymm22[21,20],zero,ymm22[22],zero,ymm22[24],zero,ymm22[22,23],zero,ymm22[25] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,3,3] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm7 = ymm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,ymm20[19],zero,ymm20[21],zero,zero,ymm20[20],zero,ymm20[22],zero,ymm20[24],zero,zero,ymm20[23],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3] +; AVX512BW-FAST-NEXT: vpor %ymm2, %ymm7, %ymm2 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1 +; AVX512BW-FAST-NEXT: vpshufb %ymm15, %ymm17, %ymm2 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[2,2,3,3] -; AVX512BW-FAST-NEXT: vpshufb %ymm16, %ymm20, %ymm7 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm7 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm24[21],zero,zero,ymm24[20],zero,ymm24[22],zero,ymm24[24],zero,zero,ymm24[23],zero,ymm24[25],zero,zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm7 = ymm7[2,2,3,3] ; AVX512BW-FAST-NEXT: vpor %ymm2, %ymm7, %ymm2 -; AVX512BW-FAST-NEXT: vpermd %ymm20, %ymm3, %ymm3 -; AVX512BW-FAST-NEXT: vpshufb %ymm4, %ymm18, %ymm3 {%k1} +; AVX512BW-FAST-NEXT: vpermd %ymm24, %ymm3, %ymm3 +; AVX512BW-FAST-NEXT: vpshufb %ymm4, %ymm17, %ymm3 {%k1} ; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm2 {%k2} ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm1 = <3,3,3,3,u,4,4,4,12,14,13,13,13,13,12,14> @@ -4574,7 +4572,7 @@ define void @store_i8_stride5_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm2 {%k1} ; AVX512BW-FAST-NEXT: vmovdqa64 %zmm2, 64(%r9) ; AVX512BW-FAST-NEXT: vmovdqa64 %zmm6, (%r9) -; AVX512BW-FAST-NEXT: vmovdqa64 %zmm17, 128(%r9) +; AVX512BW-FAST-NEXT: vmovdqa64 %zmm16, 128(%r9) ; AVX512BW-FAST-NEXT: vmovdqa64 %zmm13, 256(%r9) ; AVX512BW-FAST-NEXT: vmovdqa64 %zmm0, 192(%r9) ; AVX512BW-FAST-NEXT: vzeroupper diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll index 3bc7b6e95822..b1493847cdb8 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-6.ll @@ -1724,10 +1724,10 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-LABEL: store_i8_stride6_vf32: ; AVX512F-SLOW: # %bb.0: ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm2 -; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %ymm3 -; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %ymm4 -; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %ymm5 +; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm3 +; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %ymm5 +; AVX512F-SLOW-NEXT: vmovdqa (%rdx), %ymm2 +; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %ymm4 ; AVX512F-SLOW-NEXT: vmovdqa (%r8), %ymm0 ; AVX512F-SLOW-NEXT: vmovdqa (%r9), %ymm1 ; AVX512F-SLOW-NEXT: vmovdqa (%rcx), %xmm7 @@ -1735,7 +1735,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[10,11,8,9,6,7,12,13,14,15,14,15,14,15,14,15] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,0,0,1] -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm9 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[16],ymm5[16],ymm4[17],ymm5[17],ymm4[18],ymm5[18],ymm4[19],ymm5[19],ymm4[20],ymm5[20],ymm4[21],ymm5[21],ymm4[22],ymm5[22],ymm4[23],ymm5[23] +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm9 = ymm2[0],ymm4[0],ymm2[1],ymm4[1],ymm2[2],ymm4[2],ymm2[3],ymm4[3],ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[16],ymm4[16],ymm2[17],ymm4[17],ymm2[18],ymm4[18],ymm2[19],ymm4[19],ymm2[20],ymm4[20],ymm2[21],ymm4[21],ymm2[22],ymm4[22],ymm2[23],ymm4[23] ; AVX512F-SLOW-NEXT: vprold $16, %ymm9, %ymm9 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm9 = ymm9[2,2,2,3] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm6, %zmm6 @@ -1744,13 +1744,13 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm11 = xmm10[8],xmm9[8],xmm10[9],xmm9[9],xmm10[10],xmm9[10],xmm10[11],xmm9[11],xmm10[12],xmm9[12],xmm10[13],xmm9[13],xmm10[14],xmm9[14],xmm10[15],xmm9[15] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm11 = ymm11[0,0,0,1] -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm12 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23] +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm12 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[16],ymm5[16],ymm3[17],ymm5[17],ymm3[18],ymm5[18],ymm3[19],ymm5[19],ymm3[20],ymm5[20],ymm3[21],ymm5[21],ymm3[22],ymm5[22],ymm3[23],ymm5[23] ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} ymm12 = ymm12[0,3,2,1,4,5,6,7,8,11,10,9,12,13,14,15] ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm12 = ymm12[0,1,2,3,4,5,6,5,8,9,10,11,12,13,14,13] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,2,2,3] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm13 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm14 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535] -; AVX512F-SLOW-NEXT: vpternlogq $226, %zmm6, %zmm14, %zmm13 +; AVX512F-SLOW-NEXT: vpternlogd $226, %zmm6, %zmm14, %zmm13 ; AVX512F-SLOW-NEXT: vmovdqa (%r9), %xmm11 ; AVX512F-SLOW-NEXT: vmovdqa (%r8), %xmm12 ; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm12[8],xmm11[8],xmm12[9],xmm11[9],xmm12[10],xmm11[10],xmm12[11],xmm11[11],xmm12[12],xmm11[12],xmm12[13],xmm11[13],xmm12[14],xmm11[14],xmm12[15],xmm11[15] @@ -1761,7 +1761,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} ymm15 = ymm15[0,1,2,3,4,4,4,4,8,9,10,11,12,12,12,12] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm15 = ymm15[2,2,2,3] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm15, %zmm6, %zmm6 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm6 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm6 ; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} xmm13 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0] ; AVX512F-SLOW-NEXT: vpshufb %xmm13, %xmm9, %xmm15 ; AVX512F-SLOW-NEXT: vpshufb %xmm13, %xmm10, %xmm13 @@ -1769,17 +1769,17 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm15 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7] ; AVX512F-SLOW-NEXT: vprold $16, %xmm15, %xmm15 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm13, %zmm15, %zmm13 -; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm15 = zmm13[0,0,0,1,4,4,4,5] -; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm13 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10] -; AVX512F-SLOW-NEXT: vpshufb %xmm13, %xmm7, %xmm7 -; AVX512F-SLOW-NEXT: vpshufb %xmm13, %xmm8, %xmm8 +; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm13 = zmm13[0,0,0,1,4,4,4,5] +; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm15 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10] +; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm7, %xmm7 +; AVX512F-SLOW-NEXT: vpshufb %xmm15, %xmm8, %xmm8 ; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3],xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7] ; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} xmm8 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3],xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7] ; AVX512F-SLOW-NEXT: vpshuflw {{.*#+}} xmm8 = xmm8[0,3,2,1,4,5,6,7] ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm8 = xmm8[0,1,2,3,4,5,6,5] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm7 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm8 = zmm7[0,0,0,1,4,4,4,5] -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm8 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm13, %zmm8 ; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm7 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0] ; AVX512F-SLOW-NEXT: vpshufb %xmm7, %xmm11, %xmm9 ; AVX512F-SLOW-NEXT: vpshufb %xmm7, %xmm12, %xmm10 @@ -1789,33 +1789,31 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm10 = xmm10[0,1,2,3,4,4,4,4] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm10, %zmm9 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm9 = zmm9[0,0,0,1,4,4,4,5] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm8, %zmm14, %zmm9 -; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm8 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0] -; AVX512F-SLOW-NEXT: vpshufb %ymm8, %ymm3, %ymm10 -; AVX512F-SLOW-NEXT: vpshufb %ymm8, %ymm2, %ymm8 -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm8 = ymm8[0],ymm10[0],ymm8[1],ymm10[1],ymm8[2],ymm10[2],ymm8[3],ymm10[3],ymm8[4],ymm10[4],ymm8[5],ymm10[5],ymm8[6],ymm10[6],ymm8[7],ymm10[7],ymm8[16],ymm10[16],ymm8[17],ymm10[17],ymm8[18],ymm10[18],ymm8[19],ymm10[19],ymm8[20],ymm10[20],ymm8[21],ymm10[21],ymm8[22],ymm10[22],ymm8[23],ymm10[23] -; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} ymm10 = ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15],ymm4[24],ymm5[24],ymm4[25],ymm5[25],ymm4[26],ymm5[26],ymm4[27],ymm5[27],ymm4[28],ymm5[28],ymm4[29],ymm5[29],ymm4[30],ymm5[30],ymm4[31],ymm5[31] -; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31] +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm8, %zmm14, %zmm9 +; AVX512F-SLOW-NEXT: vpshufb %ymm15, %ymm4, %ymm8 +; AVX512F-SLOW-NEXT: vpshufb %ymm15, %ymm2, %ymm10 +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm8 = ymm10[0],ymm8[0],ymm10[1],ymm8[1],ymm10[2],ymm8[2],ymm10[3],ymm8[3],ymm10[4],ymm8[4],ymm10[5],ymm8[5],ymm10[6],ymm8[6],ymm10[7],ymm8[7],ymm10[16],ymm8[16],ymm10[17],ymm8[17],ymm10[18],ymm8[18],ymm10[19],ymm8[19],ymm10[20],ymm8[20],ymm10[21],ymm8[21],ymm10[22],ymm8[22],ymm10[23],ymm8[23] +; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} ymm10 = ymm3[8],ymm5[8],ymm3[9],ymm5[9],ymm3[10],ymm5[10],ymm3[11],ymm5[11],ymm3[12],ymm5[12],ymm3[13],ymm5[13],ymm3[14],ymm5[14],ymm3[15],ymm5[15],ymm3[24],ymm5[24],ymm3[25],ymm5[25],ymm3[26],ymm5[26],ymm3[27],ymm5[27],ymm3[28],ymm5[28],ymm3[29],ymm5[29],ymm3[30],ymm5[30],ymm3[31],ymm5[31] +; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm10 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm8, %zmm8 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm8 = zmm8[2,2,2,3,6,6,6,7] -; AVX512F-SLOW-NEXT: vpshufb %ymm13, %ymm5, %ymm5 -; AVX512F-SLOW-NEXT: vpshufb %ymm13, %ymm4, %ymm4 -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[16],ymm5[16],ymm4[17],ymm5[17],ymm4[18],ymm5[18],ymm4[19],ymm5[19],ymm4[20],ymm5[20],ymm4[21],ymm5[21],ymm4[22],ymm5[22],ymm4[23],ymm5[23] -; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31] -; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31] -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm4, %zmm2 +; AVX512F-SLOW-NEXT: vpbroadcastq {{.*#+}} ymm10 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0] +; AVX512F-SLOW-NEXT: vpshufb %ymm10, %ymm5, %ymm5 +; AVX512F-SLOW-NEXT: vpshufb %ymm10, %ymm3, %ymm3 +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm5[0],ymm3[1],ymm5[1],ymm3[2],ymm5[2],ymm3[3],ymm5[3],ymm3[4],ymm5[4],ymm3[5],ymm5[5],ymm3[6],ymm5[6],ymm3[7],ymm5[7],ymm3[16],ymm5[16],ymm3[17],ymm5[17],ymm3[18],ymm5[18],ymm3[19],ymm5[19],ymm3[20],ymm5[20],ymm3[21],ymm5[21],ymm3[22],ymm5[22],ymm3[23],ymm5[23] +; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm2[8],ymm4[8],ymm2[9],ymm4[9],ymm2[10],ymm4[10],ymm2[11],ymm4[11],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15],ymm2[24],ymm4[24],ymm2[25],ymm4[25],ymm2[26],ymm4[26],ymm2[27],ymm4[27],ymm2[28],ymm4[28],ymm2[29],ymm4[29],ymm2[30],ymm4[30],ymm2[31],ymm4[31] +; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm2 = ymm2[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31] +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm2 = zmm2[2,2,2,3,6,6,6,7] -; AVX512F-SLOW-NEXT: vbroadcasti64x4 {{.*#+}} zmm3 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535] -; AVX512F-SLOW-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3] -; AVX512F-SLOW-NEXT: vpternlogq $202, %zmm8, %zmm2, %zmm3 -; AVX512F-SLOW-NEXT: vpshufb %ymm7, %ymm1, %ymm2 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm2 +; AVX512F-SLOW-NEXT: vpshufb %ymm7, %ymm1, %ymm3 ; AVX512F-SLOW-NEXT: vpshufb %ymm7, %ymm0, %ymm4 -; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm4[0],ymm2[0],ymm4[1],ymm2[1],ymm4[2],ymm2[2],ymm4[3],ymm2[3],ymm4[4],ymm2[4],ymm4[5],ymm2[5],ymm4[6],ymm2[6],ymm4[7],ymm2[7],ymm4[16],ymm2[16],ymm4[17],ymm2[17],ymm4[18],ymm2[18],ymm4[19],ymm2[19],ymm4[20],ymm2[20],ymm4[21],ymm2[21],ymm4[22],ymm2[22],ymm4[23],ymm2[23] +; AVX512F-SLOW-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[16],ymm3[16],ymm4[17],ymm3[17],ymm4[18],ymm3[18],ymm4[19],ymm3[19],ymm4[20],ymm3[20],ymm4[21],ymm3[21],ymm4[22],ymm3[22],ymm4[23],ymm3[23] ; AVX512F-SLOW-NEXT: vpunpckhbw {{.*#+}} ymm0 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,26,27,24,25,22,23,28,29,26,27,28,29,30,31] -; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0 +; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm0 = zmm0[2,2,2,3,6,6,6,7] -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, 128(%rax) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm9, (%rax) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm6, 64(%rax) @@ -1831,34 +1829,32 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vmovdqa (%rcx), %ymm5 ; AVX512F-FAST-NEXT: vmovdqa (%r8), %ymm0 ; AVX512F-FAST-NEXT: vmovdqa (%r9), %ymm1 -; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm6 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0] -; AVX512F-FAST-NEXT: vpshufb %ymm6, %ymm3, %ymm7 -; AVX512F-FAST-NEXT: vpshufb %ymm6, %ymm2, %ymm6 -; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm6 = ymm6[0],ymm7[0],ymm6[1],ymm7[1],ymm6[2],ymm7[2],ymm6[3],ymm7[3],ymm6[4],ymm7[4],ymm6[5],ymm7[5],ymm6[6],ymm7[6],ymm6[7],ymm7[7],ymm6[16],ymm7[16],ymm6[17],ymm7[17],ymm6[18],ymm7[18],ymm6[19],ymm7[19],ymm6[20],ymm7[20],ymm6[21],ymm7[21],ymm6[22],ymm7[22],ymm6[23],ymm7[23] -; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} ymm7 = ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15],ymm4[24],ymm5[24],ymm4[25],ymm5[25],ymm4[26],ymm5[26],ymm4[27],ymm5[27],ymm4[28],ymm5[28],ymm4[29],ymm5[29],ymm4[30],ymm5[30],ymm4[31],ymm5[31] -; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm7 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31] -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm6, %zmm6 -; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm6 = zmm6[2,2,2,3,6,6,6,7] ; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm7 = [5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10,5,8,7,6,9,0,0,10] -; AVX512F-FAST-NEXT: vpshufb %ymm7, %ymm5, %ymm8 -; AVX512F-FAST-NEXT: vpshufb %ymm7, %ymm4, %ymm9 -; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm8 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[16],ymm8[16],ymm9[17],ymm8[17],ymm9[18],ymm8[18],ymm9[19],ymm8[19],ymm9[20],ymm8[20],ymm9[21],ymm8[21],ymm9[22],ymm8[22],ymm9[23],ymm8[23] -; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} ymm9 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31] -; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31] +; AVX512F-FAST-NEXT: vpshufb %ymm7, %ymm5, %ymm6 +; AVX512F-FAST-NEXT: vpshufb %ymm7, %ymm4, %ymm8 +; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm6 = ymm8[0],ymm6[0],ymm8[1],ymm6[1],ymm8[2],ymm6[2],ymm8[3],ymm6[3],ymm8[4],ymm6[4],ymm8[5],ymm6[5],ymm8[6],ymm6[6],ymm8[7],ymm6[7],ymm8[16],ymm6[16],ymm8[17],ymm6[17],ymm8[18],ymm6[18],ymm8[19],ymm6[19],ymm8[20],ymm6[20],ymm8[21],ymm6[21],ymm8[22],ymm6[22],ymm8[23],ymm6[23] +; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} ymm8 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31] +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,22,23,28,29,26,27,30,31,30,31,30,31,30,31] +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm8, %zmm6, %zmm6 +; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm6 = zmm6[2,2,2,3,6,6,6,7] +; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm8 = [8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0,8,7,6,9,0,0,10,0] +; AVX512F-FAST-NEXT: vpshufb %ymm8, %ymm3, %ymm9 +; AVX512F-FAST-NEXT: vpshufb %ymm8, %ymm2, %ymm8 +; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm8 = ymm8[0],ymm9[0],ymm8[1],ymm9[1],ymm8[2],ymm9[2],ymm8[3],ymm9[3],ymm8[4],ymm9[4],ymm8[5],ymm9[5],ymm8[6],ymm9[6],ymm8[7],ymm9[7],ymm8[16],ymm9[16],ymm8[17],ymm9[17],ymm8[18],ymm9[18],ymm8[19],ymm9[19],ymm8[20],ymm9[20],ymm8[21],ymm9[21],ymm8[22],ymm9[22],ymm8[23],ymm9[23] +; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} ymm9 = ymm4[8],ymm5[8],ymm4[9],ymm5[9],ymm4[10],ymm5[10],ymm4[11],ymm5[11],ymm4[12],ymm5[12],ymm4[13],ymm5[13],ymm4[14],ymm5[14],ymm4[15],ymm5[15],ymm4[24],ymm5[24],ymm4[25],ymm5[25],ymm4[26],ymm5[26],ymm4[27],ymm5[27],ymm4[28],ymm5[28],ymm4[29],ymm5[29],ymm4[30],ymm5[30],ymm4[31],ymm5[31] +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,26,27,24,25,22,23,28,29,30,31,30,31,30,31,30,31] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm8, %zmm8 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm8 = zmm8[2,2,2,3,6,6,6,7] -; AVX512F-FAST-NEXT: vbroadcasti64x4 {{.*#+}} zmm9 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535] -; AVX512F-FAST-NEXT: # zmm9 = mem[0,1,2,3,0,1,2,3] -; AVX512F-FAST-NEXT: vpternlogq $202, %zmm6, %zmm8, %zmm9 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm8 ; AVX512F-FAST-NEXT: vpbroadcastq {{.*#+}} ymm10 = [6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0,6,5,8,7,0,9,0,0] ; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm1, %ymm6 -; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm0, %ymm8 -; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm6 = ymm8[0],ymm6[0],ymm8[1],ymm6[1],ymm8[2],ymm6[2],ymm8[3],ymm6[3],ymm8[4],ymm6[4],ymm8[5],ymm6[5],ymm8[6],ymm6[6],ymm8[7],ymm6[7],ymm8[16],ymm6[16],ymm8[17],ymm6[17],ymm8[18],ymm6[18],ymm8[19],ymm6[19],ymm8[20],ymm6[20],ymm8[21],ymm6[21],ymm8[22],ymm6[22],ymm8[23],ymm6[23] -; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} ymm8 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] -; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm8 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,26,27,24,25,22,23,28,29,26,27,28,29,30,31] -; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm8, %zmm6, %zmm6 +; AVX512F-FAST-NEXT: vpshufb %ymm10, %ymm0, %ymm9 +; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm6 = ymm9[0],ymm6[0],ymm9[1],ymm6[1],ymm9[2],ymm6[2],ymm9[3],ymm6[3],ymm9[4],ymm6[4],ymm9[5],ymm6[5],ymm9[6],ymm6[6],ymm9[7],ymm6[7],ymm9[16],ymm6[16],ymm9[17],ymm6[17],ymm9[18],ymm6[18],ymm9[19],ymm6[19],ymm9[20],ymm6[20],ymm9[21],ymm6[21],ymm9[22],ymm6[22],ymm9[23],ymm6[23] +; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} ymm9 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31] +; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm9 = ymm9[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,26,27,24,25,22,23,28,29,26,27,28,29,30,31] +; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm6, %zmm6 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm6 = zmm6[2,2,2,3,6,6,6,7] -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm6 ; AVX512F-FAST-NEXT: vmovdqa (%rcx), %xmm9 ; AVX512F-FAST-NEXT: vpshufb %xmm7, %xmm9, %xmm8 ; AVX512F-FAST-NEXT: vmovdqa (%rdx), %xmm11 @@ -1878,7 +1874,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vprold $16, %xmm14, %xmm14 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm13, %zmm14, %zmm13 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm14 = zmm13[0,0,0,1,4,4,4,5] -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm14 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm12, %zmm14 ; AVX512F-FAST-NEXT: vmovdqa (%r9), %xmm12 ; AVX512F-FAST-NEXT: vpshufb %xmm10, %xmm12, %xmm15 ; AVX512F-FAST-NEXT: vmovdqa (%r8), %xmm13 @@ -1889,7 +1885,7 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm10, %zmm15, %zmm10 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm10 = zmm10[0,0,0,1,4,4,4,5] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm15 = [65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535,0,65535,65535] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm14, %zmm15, %zmm10 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm14, %zmm15, %zmm10 ; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm4 = ymm4[0],ymm5[0],ymm4[1],ymm5[1],ymm4[2],ymm5[2],ymm4[3],ymm5[3],ymm4[4],ymm5[4],ymm4[5],ymm5[5],ymm4[6],ymm5[6],ymm4[7],ymm5[7],ymm4[16],ymm5[16],ymm4[17],ymm5[17],ymm4[18],ymm5[18],ymm4[19],ymm5[19],ymm4[20],ymm5[20],ymm4[21],ymm5[21],ymm4[22],ymm5[22],ymm4[23],ymm5[23] ; AVX512F-FAST-NEXT: vprold $16, %ymm4, %ymm4 ; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm11[8],xmm9[8],xmm11[9],xmm9[9],xmm11[10],xmm9[10],xmm11[11],xmm9[11],xmm11[12],xmm9[12],xmm11[13],xmm9[13],xmm11[14],xmm9[14],xmm11[15],xmm9[15] @@ -1901,13 +1897,13 @@ define void @store_i8_stride6_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm8[8],xmm7[8],xmm8[9],xmm7[9],xmm8[10],xmm7[10],xmm8[11],xmm7[11],xmm8[12],xmm7[12],xmm8[13],xmm7[13],xmm8[14],xmm7[14],xmm8[15],xmm7[15] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[8,9,6,7,12,13,10,11,14,15,14,15,14,15,14,15] ; AVX512F-FAST-NEXT: vpermt2q %zmm2, %zmm9, %zmm3 -; AVX512F-FAST-NEXT: vpternlogq $226, %zmm5, %zmm15, %zmm3 +; AVX512F-FAST-NEXT: vpternlogd $226, %zmm5, %zmm15, %zmm3 ; AVX512F-FAST-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20,21,18,19,16,17,22,23,24,25,24,25,24,25,24,25] ; AVX512F-FAST-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm13[8],xmm12[8],xmm13[9],xmm12[9],xmm13[10],xmm12[10],xmm13[11],xmm12[11],xmm13[12],xmm12[12],xmm13[13],xmm12[13],xmm13[14],xmm12[14],xmm13[15],xmm12[15] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[4,5,10,11,8,9,6,7,12,13,10,11,12,13,14,15] ; AVX512F-FAST-NEXT: vpermt2q %zmm0, %zmm9, %zmm1 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm1 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm3, %zmm1 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, 64(%rax) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm10, (%rax) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm6, 128(%rax) @@ -4306,7 +4302,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm28[0,1,2,3],zmm1[4,5,6,7] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm13, %zmm1 ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm8 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm5, %zmm8, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm5, %zmm8, %zmm1 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm5 = ymm29[0,0,0,1] ; AVX512F-SLOW-NEXT: vprold $16, %ymm30, %ymm9 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm11 = ymm21[0,0,0,1] @@ -4336,7 +4332,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm2 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm15[0,1,2,3],zmm2[4,5,6,7] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm29, %zmm28, %zmm5 -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm8, %zmm5 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm2, %zmm8, %zmm5 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm2 = zmm27[0,0,0,1,4,4,4,5] ; AVX512F-SLOW-NEXT: vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm8 = mem[0,0,0,1,4,4,4,5] @@ -4350,7 +4346,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm8 = mem[0,0,0,1,4,4,4,5] ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm10 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm10, %zmm8 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm2, %zmm10, %zmm8 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm2 = zmm26[0,0,0,1,4,4,4,5] ; AVX512F-SLOW-NEXT: vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm11 = mem[0,0,0,1,4,4,4,5] @@ -4362,7 +4358,7 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm31[0,1,2,3],zmm2[4,5,6,7] ; AVX512F-SLOW-NEXT: vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm11 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm11 = mem[0,0,0,1,4,4,4,5] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm2, %zmm10, %zmm11 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm2, %zmm10, %zmm11 ; AVX512F-SLOW-NEXT: vpermq $234, (%rsp), %zmm2 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm2 = mem[2,2,2,3,6,6,6,7] ; AVX512F-SLOW-NEXT: vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10 # 64-byte Folded Reload @@ -4382,11 +4378,11 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm2 = mem[2,2,2,3,6,6,6,7] ; AVX512F-SLOW-NEXT: vmovdqa64 {{.*#+}} zmm6 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm6, %zmm2 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm6, %zmm2 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm4[0,1,2,3],zmm0[4,5,6,7] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm3 = zmm23[2,2,2,3,6,6,6,7] -; AVX512F-SLOW-NEXT: vpternlogq $184, %zmm0, %zmm6, %zmm3 +; AVX512F-SLOW-NEXT: vpternlogd $184, %zmm0, %zmm6, %zmm3 ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm3, 128(%rax) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm2, 320(%rax) @@ -4605,13 +4601,13 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm0, %zmm7 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm7 = zmm13[0,1,2,3],zmm7[4,5,6,7] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm8 = [255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm7, %zmm8, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm7, %zmm8, %zmm6 ; AVX512F-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-FAST-NEXT: vmovdqa64 %zmm6, 256(%rax) ; AVX512F-FAST-NEXT: vpternlogq $184, %ymm11, %ymm16, %ymm12 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm12, %zmm0, %zmm6 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm2[0,1,2,3],zmm6[4,5,6,7] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm2, %zmm8, %zmm3 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm2, %zmm8, %zmm3 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm3, 64(%rax) ; AVX512F-FAST-NEXT: vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Folded Reload ; AVX512F-FAST-NEXT: # zmm2 = mem[2,2,2,3,6,6,6,7] @@ -4634,13 +4630,13 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpermq $234, {{[-0-9]+}}(%r{{[sb]}}p), %zmm7 # 64-byte Folded Reload ; AVX512F-FAST-NEXT: # zmm7 = mem[2,2,2,3,6,6,6,7] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm8 = [255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm3, %zmm8, %zmm7 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm3, %zmm8, %zmm7 ; AVX512F-FAST-NEXT: vpternlogq $184, %ymm6, %ymm2, %ymm31 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm30, %zmm0, %zmm3 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm3 = zmm31[0,1,2,3],zmm3[4,5,6,7] ; AVX512F-FAST-NEXT: vpermq $234, (%rsp), %zmm6 # 64-byte Folded Reload ; AVX512F-FAST-NEXT: # zmm6 = mem[2,2,2,3,6,6,6,7] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm3, %zmm8, %zmm6 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm3, %zmm8, %zmm6 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm23[0,0,0,1,4,4,4,5] ; AVX512F-FAST-NEXT: vpermq $64, {{[-0-9]+}}(%r{{[sb]}}p), %zmm8 # 64-byte Folded Reload ; AVX512F-FAST-NEXT: # zmm8 = mem[0,0,0,1,4,4,4,5] @@ -4658,11 +4654,11 @@ define void @store_i8_stride6_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm2 = zmm5[0,1,2,3],zmm2[4,5,6,7] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm26[0,0,0,1,4,4,4,5] ; AVX512F-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255,255] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm2, %zmm4, %zmm3 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm2, %zmm4, %zmm3 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm0, %zmm0 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm28[0,0,0,1,4,4,4,5] -; AVX512F-FAST-NEXT: vpternlogq $184, %zmm0, %zmm4, %zmm1 +; AVX512F-FAST-NEXT: vpternlogd $184, %zmm0, %zmm4, %zmm1 ; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm3, 192(%rax) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm6, 128(%rax) diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll index 54ed0f184827..04c1bd532252 100644 --- a/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll +++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i8-stride-7.ll @@ -659,7 +659,7 @@ define void @store_i8_stride7_vf8(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vecp ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[4,12],zero,ymm1[u,u,u,u,5,13],zero,ymm1[u,u,u,u,6,14,22,u,u,u,u],zero,zero,ymm1[23,u,u,u,u,u,u,u,u] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 ; AVX512F-SLOW-NEXT: vporq %zmm0, %zmm1, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-SLOW-NEXT: vextracti32x4 $2, %zmm0, 32(%rax) ; AVX512F-SLOW-NEXT: vextracti32x4 $3, %zmm0, %xmm1 ; AVX512F-SLOW-NEXT: vmovq %xmm1, 48(%rax) @@ -1560,7 +1560,7 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[u,u,u,u,0,8],zero,ymm0[u,u,u,u,1,9],zero,ymm0[u,u,u,u,18,26],zero,ymm0[u,u,u,u,19,27],zero,ymm0[u,u,u,u] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm7, %zmm0, %zmm0 ; AVX512F-SLOW-NEXT: vporq %zmm1, %zmm0, %zmm0 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-SLOW-NEXT: vmovdqa %xmm5, 96(%rax) ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, (%rax) ; AVX512F-SLOW-NEXT: vmovdqa %ymm4, 64(%rax) @@ -1631,7 +1631,7 @@ define void @store_i8_stride7_vf16(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm1[0,4],zero,ymm1[u,u,u,u,1,5],zero,ymm1[u,u,u,u,2,6],zero,ymm1[u,u,u,u,19,23],zero,ymm1[u,u,u,u,24,28],zero,ymm1[u] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm6, %zmm1 ; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm1 ; AVX512F-FAST-NEXT: vmovdqa %xmm3, 96(%rax) ; AVX512F-FAST-NEXT: vmovdqa64 %zmm1, (%rax) ; AVX512F-FAST-NEXT: vmovdqa %ymm2, 64(%rax) @@ -3342,7 +3342,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm8, %zmm8 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm8 = zmm8[2,3,2,3,6,7,6,7] ; AVX512F-SLOW-NEXT: vporq %zmm7, %zmm8, %zmm7 -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm7 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm7 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm8 = ymm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25,u,23,u,u,u,u,26,u,24,u,u,u,u,27,u] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm8 = ymm8[2,3,2,3] ; AVX512F-SLOW-NEXT: vbroadcasti32x4 {{.*#+}} ymm16 = [18374967954648269055,71777218572844800,18374967954648269055,71777218572844800] @@ -3371,7 +3371,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm11, %zmm10, %zmm10 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm10 = zmm10[2,3,2,3,6,7,6,7] ; AVX512F-SLOW-NEXT: vporq %zmm10, %zmm8, %zmm8 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm8 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm8 ; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm8 ; AVX512F-SLOW-NEXT: vmovdqa (%rsi), %xmm11 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm11[u],zero,xmm11[7],zero,xmm11[5,u,u,u],zero,xmm11[8],zero,xmm11[6,u,u,u],zero @@ -3391,7 +3391,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm14 = xmm14[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9] ; AVX512F-SLOW-NEXT: vinserti32x4 $2, %xmm13, %zmm14, %zmm13 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm18 = zmm13[0,1,0,1,4,5,4,5] -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm18 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm18 ; AVX512F-SLOW-NEXT: vmovdqa (%r9), %xmm13 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm10 = zero,xmm13[4,u,u,u],zero,xmm13[7],zero,xmm13[5,u,u,u],zero,xmm13[8],zero,xmm13[6] ; AVX512F-SLOW-NEXT: vmovdqa (%r8), %xmm14 @@ -3407,7 +3407,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,2,0] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm10, %zmm0, %zmm0 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm10 = zmm0[0,0,1,0,4,4,5,4] -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm19, %zmm10 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm19, %zmm10 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm10 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm1[14,u,u],zero,zero,zero,zero,ymm1[15,u,u],zero,zero,zero,zero,ymm1[16,u,u],zero,zero,zero,zero,ymm1[17,u,u],zero,zero,zero,zero,ymm1[18] ; AVX512F-SLOW-NEXT: vmovdqa64 %ymm1, %ymm19 @@ -3426,7 +3426,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm7, %zmm1 -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = zero,ymm5[u,u,u,u,u,14],zero,ymm5[u,u,u,u,u,15],zero,ymm5[u,u,u,u,u,16],zero,ymm5[u,u,u,u,u,17],zero,ymm5[u,u,u] ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm7 = ymm6[13,u,u,u,u,u],zero,ymm6[14,u,u,u,u,u],zero,ymm6[15,u,u,u,u,u],zero,ymm6[16,u,u,u,u,u],zero,ymm6[17,u,u,u] ; AVX512F-SLOW-NEXT: vpor %ymm0, %ymm7, %ymm0 @@ -3437,8 +3437,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpshufhw {{.*#+}} xmm7 = xmm15[0,1,2,3,4,5,5,6] ; AVX512F-SLOW-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[2,2,3,3] ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm7 = ymm7[0,1,0,1] -; AVX512F-SLOW-NEXT: vmovdqa {{.*#+}} ymm9 = [255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255] -; AVX512F-SLOW-NEXT: vpandn %ymm7, %ymm9, %ymm7 +; AVX512F-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm7, %ymm7 ; AVX512F-SLOW-NEXT: vpshufb {{.*#+}} ymm9 = zero,ymm4[13,u,u,u,u],zero,zero,ymm4[14,u,u,u,u],zero,zero,ymm4[15,u,u,u,u],zero,zero,ymm4[16,u,u,u,u],zero,zero,ymm4[17,u,u] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm9, %zmm7, %zmm7 ; AVX512F-SLOW-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm7 @@ -3502,7 +3501,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm13 = xmm13[4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9] ; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm10, %zmm13, %zmm10 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm15 = zmm10[0,1,0,1,4,5,4,5] -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm15 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm15 ; AVX512F-FAST-NEXT: vmovdqa (%r10), %xmm10 ; AVX512F-FAST-NEXT: vpshuflw {{.*#+}} xmm7 = xmm10[1,1,0,0,4,5,6,7] ; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm13 = @@ -3519,7 +3518,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[u,u,4,5,0,1,u,u,u,6,7,2,3,u,u,u] ; AVX512F-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm7, %zmm0 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm7 = zmm0[0,1,0,1,4,5,4,5] -; AVX512F-FAST-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm16, %zmm7 +; AVX512F-FAST-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm16, %zmm7 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm15, %zmm7 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm0 = zero,zero,zero,ymm4[14,u,u],zero,zero,zero,zero,ymm4[15,u,u],zero,zero,zero,zero,ymm4[16,u,u],zero,zero,zero,zero,ymm4[17,u,u],zero,zero,zero,zero,ymm4[18] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm15 = ymm2[0,1,14],zero,ymm2[u,u,0,1,14,15],zero,ymm2[u,u,13,2,3,16],zero,ymm2[u,u,28,29,16,17],zero,ymm2[u,u,19,28,29,18],zero @@ -3535,7 +3534,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm8, %zmm9 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm9 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm9 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm0 = zero,ymm1[u,u,u,u,u,14],zero,ymm1[u,u,u,u,u,15],zero,ymm1[u,u,u,u,u,16],zero,ymm1[u,u,u,u,u,17],zero,ymm1[u,u,u] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm8 = ymm3[13,u,u,u,u,u],zero,ymm3[14,u,u,u,u,u],zero,ymm3[15,u,u,u,u,u],zero,ymm3[16,u,u,u,u,u],zero,ymm3[17,u,u,u] ; AVX512F-FAST-NEXT: vpor %ymm0, %ymm8, %ymm0 @@ -3547,8 +3546,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm10 = [2,2,3,3,2,2,3,3] ; AVX512F-FAST-NEXT: # ymm10 = mem[0,1,0,1] ; AVX512F-FAST-NEXT: vpermd %ymm8, %ymm10, %ymm8 -; AVX512F-FAST-NEXT: vmovdqa {{.*#+}} ymm10 = [255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255,255,0,255,255,255,255,255] -; AVX512F-FAST-NEXT: vpandn %ymm8, %ymm10, %ymm8 +; AVX512F-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm8, %ymm8 ; AVX512F-FAST-NEXT: vmovdqa64 %ymm17, %ymm13 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm10 = zero,ymm13[13,u,u,u,u],zero,zero,ymm13[14,u,u,u,u],zero,zero,ymm13[15,u,u,u,u],zero,zero,ymm13[16,u,u,u,u],zero,zero,ymm13[17,u,u] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm8, %zmm8 @@ -3576,7 +3574,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm11 ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm11 = zmm11[2,3,2,3,6,7,6,7] ; AVX512F-FAST-NEXT: vporq %zmm10, %zmm11, %zmm10 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm10 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm10 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm3[20],zero,ymm3[18],zero,zero,zero,zero,ymm3[21],zero,ymm3[19],zero,zero,zero,zero,ymm3[22] ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm11 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm3[25],zero,ymm3[23],zero,zero,zero,zero,ymm3[26],zero,ymm3[24],zero,zero ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm0, %zmm0 @@ -3593,7 +3591,7 @@ define void @store_i8_stride7_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm12 = ymm13[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm12 = ymm12[2,3,2,3] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm12, %zmm11, %zmm11 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm11 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm11 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm10, %zmm11 ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm6[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27,u,u,u,u,30,u,28,u,u,u,u,31,u,29,u] ; AVX512F-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3] @@ -7623,19 +7621,19 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm3 = ymm14[0,0,1,0] ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm3, %zmm13, %zmm3 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm6 # 64-byte Reload -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm3 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm6, %zmm3 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm3 ; AVX512F-SLOW-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm4 # 64-byte Folded Reload ; AVX512F-SLOW-NEXT: # zmm4 = mem[2,3,2,3,6,7,6,7] -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm4 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm4 ; AVX512F-SLOW-NEXT: vporq %ymm15, %ymm18, %ymm5 ; AVX512F-SLOW-NEXT: vporq %ymm19, %ymm20, %ymm6 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm5 ; AVX512F-SLOW-NEXT: vshufi64x2 {{.*#+}} zmm5 = zmm6[0,1,2,3],zmm5[4,5,6,7] -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm16 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm16 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm16 ; AVX512F-SLOW-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload -; AVX512F-SLOW-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm26 +; AVX512F-SLOW-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm26 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm17, %zmm26 ; AVX512F-SLOW-NEXT: vporq %ymm21, %ymm22, %ymm1 ; AVX512F-SLOW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1 @@ -7648,7 +7646,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm1 = zmm12[0,1,0,1,4,5,4,5] ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1 ; AVX512F-SLOW-NEXT: vpermq {{.*#+}} zmm0 = zmm7[0,0,1,0,4,4,5,4] -; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm0 +; AVX512F-SLOW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm11, %zmm0 ; AVX512F-SLOW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0 ; AVX512F-SLOW-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-SLOW-NEXT: vmovdqa64 %zmm0, (%rax) @@ -8039,7 +8037,7 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpshufb {{.*#+}} ymm4 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,22,23,26,27,24,25,22,23,24,25,26,27,26,27,24,25] ; AVX512F-FAST-NEXT: vpermq $238, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22 # 64-byte Folded Reload ; AVX512F-FAST-NEXT: # zmm22 = mem[2,3,2,3,6,7,6,7] -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm22 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm22 ; AVX512F-FAST-NEXT: vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Folded Reload ; AVX512F-FAST-NEXT: # ymm0 = mem[0,1,0,1] ; AVX512F-FAST-NEXT: vpermq $68, {{[-0-9]+}}(%r{{[sb]}}p), %ymm23 # 32-byte Folded Reload @@ -8061,11 +8059,11 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm24 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm23, %zmm24 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm19, %zmm2, %zmm0 -; AVX512F-FAST-NEXT: vpternlogq $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm21 +; AVX512F-FAST-NEXT: vpternlogd $228, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm21 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm7, %zmm21 ; AVX512F-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm10, %zmm0 # 32-byte Folded Reload ; AVX512F-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm2 # 64-byte Reload -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm8, %zmm0 ; AVX512F-FAST-NEXT: vpor %ymm12, %ymm15, %ymm2 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm2 @@ -8078,14 +8076,14 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512F-FAST-NEXT: vpermq {{.*#+}} zmm5 = zmm31[0,1,0,1,4,5,4,5] ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm5 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm17, %zmm2 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm2 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm18, %zmm2 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm5, %zmm2 ; AVX512F-FAST-NEXT: vpor %ymm1, %ymm13, %ymm1 ; AVX512F-FAST-NEXT: vpor %ymm11, %ymm14, %ymm5 ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm1 ; AVX512F-FAST-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm5[0,1,2,3],zmm1[4,5,6,7] ; AVX512F-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm20, %zmm4 -; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm4 +; AVX512F-FAST-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm4 ; AVX512F-FAST-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm9, %zmm4 ; AVX512F-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512F-FAST-NEXT: vmovdqa64 %zmm4, 128(%rax) @@ -8415,250 +8413,253 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; ; AVX512BW-FAST-LABEL: store_i8_stride7_vf64: ; AVX512BW-FAST: # %bb.0: -; AVX512BW-FAST-NEXT: subq $200, %rsp +; AVX512BW-FAST-NEXT: subq $72, %rsp ; AVX512BW-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax -; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %zmm9 -; AVX512BW-FAST-NEXT: vmovdqu64 %zmm9, (%rsp) # 64-byte Spill -; AVX512BW-FAST-NEXT: vmovdqa64 (%rsi), %zmm5 -; AVX512BW-FAST-NEXT: vmovdqu64 %zmm5, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill -; AVX512BW-FAST-NEXT: vmovdqa64 (%rdx), %zmm2 -; AVX512BW-FAST-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill -; AVX512BW-FAST-NEXT: vmovdqa (%rax), %ymm4 -; AVX512BW-FAST-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX512BW-FAST-NEXT: vmovdqa 32(%rax), %ymm13 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %zmm7 +; AVX512BW-FAST-NEXT: vmovdqu64 %zmm7, (%rsp) # 64-byte Spill +; AVX512BW-FAST-NEXT: vmovdqa (%rax), %ymm3 +; AVX512BW-FAST-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX512BW-FAST-NEXT: vmovdqa 32(%rax), %ymm11 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} ymm0 = [12,13,2,3,12,13,0,1,14,15,2,3,0,1,14,15,28,29,18,19,28,29,16,17,30,31,18,19,16,17,30,31] -; AVX512BW-FAST-NEXT: vpshufb %ymm0, %ymm4, %ymm1 -; AVX512BW-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6] -; AVX512BW-FAST-NEXT: # ymm3 = mem[0,1,0,1] -; AVX512BW-FAST-NEXT: vpermw %ymm4, %ymm3, %ymm3 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm6 -; AVX512BW-FAST-NEXT: vmovdqa (%r9), %ymm15 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm17 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128] -; AVX512BW-FAST-NEXT: vpshufb %ymm17, %ymm15, %ymm7 -; AVX512BW-FAST-NEXT: vmovdqa (%r8), %ymm1 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm20 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128] -; AVX512BW-FAST-NEXT: vpshufb %ymm20, %ymm1, %ymm8 -; AVX512BW-FAST-NEXT: vmovdqa64 %ymm1, %ymm24 -; AVX512BW-FAST-NEXT: vpor %ymm7, %ymm8, %ymm7 -; AVX512BW-FAST-NEXT: vmovdqa64 (%r9), %xmm25 -; AVX512BW-FAST-NEXT: vmovdqa (%r8), %xmm10 -; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm8 = xmm10[8],xmm25[8],xmm10[9],xmm25[9],xmm10[10],xmm25[10],xmm10[11],xmm25[11],xmm10[12],xmm25[12],xmm10[13],xmm25[13],xmm10[14],xmm25[14],xmm10[15],xmm25[15] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm8 = xmm8[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm8 = ymm8[0,1,0,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm7, %zmm8, %zmm22 +; AVX512BW-FAST-NEXT: vpshufb %ymm0, %ymm3, %ymm1 +; AVX512BW-FAST-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [4,5,4,5,5,6,5,6,4,5,4,5,5,6,5,6] +; AVX512BW-FAST-NEXT: # ymm2 = mem[0,1,0,1] +; AVX512BW-FAST-NEXT: vpermw %ymm3, %ymm2, %ymm2 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm4 +; AVX512BW-FAST-NEXT: vmovdqa (%r9), %ymm2 +; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} ymm1 = [13,128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128] +; AVX512BW-FAST-NEXT: vpshufb %ymm1, %ymm2, %ymm5 +; AVX512BW-FAST-NEXT: vmovdqa %ymm2, %ymm3 +; AVX512BW-FAST-NEXT: vmovdqa (%r8), %ymm2 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm16 = [128,128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128] +; AVX512BW-FAST-NEXT: vpshufb %ymm16, %ymm2, %ymm6 +; AVX512BW-FAST-NEXT: vmovdqa %ymm2, %ymm12 +; AVX512BW-FAST-NEXT: vpor %ymm5, %ymm6, %ymm5 +; AVX512BW-FAST-NEXT: vmovdqa (%r9), %xmm2 +; AVX512BW-FAST-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX512BW-FAST-NEXT: vmovdqa (%r8), %xmm6 +; AVX512BW-FAST-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill +; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm6 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm6 = xmm6[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,1,0,1] +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm5, %zmm6, %zmm21 ; AVX512BW-FAST-NEXT: movabsq $2323999253380730912, %r10 # imm = 0x2040810204081020 ; AVX512BW-FAST-NEXT: kmovq %r10, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm6, %zmm22 {%k1} -; AVX512BW-FAST-NEXT: vmovdqa (%rdx), %ymm1 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm21 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31] -; AVX512BW-FAST-NEXT: vpshufb %ymm21, %ymm1, %ymm6 -; AVX512BW-FAST-NEXT: vmovdqa %ymm1, %ymm7 -; AVX512BW-FAST-NEXT: vmovdqa (%rcx), %ymm1 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm23 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128] -; AVX512BW-FAST-NEXT: vpshufb %ymm23, %ymm1, %ymm11 -; AVX512BW-FAST-NEXT: vmovdqa %ymm1, %ymm8 -; AVX512BW-FAST-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill -; AVX512BW-FAST-NEXT: vpor %ymm6, %ymm11, %ymm6 -; AVX512BW-FAST-NEXT: vmovdqa (%rdx), %xmm14 -; AVX512BW-FAST-NEXT: vmovdqa64 (%rcx), %xmm16 -; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm11 = xmm16[8],xmm14[8],xmm16[9],xmm14[9],xmm16[10],xmm14[10],xmm16[11],xmm14[11],xmm16[12],xmm14[12],xmm16[13],xmm14[13],xmm16[14],xmm14[14],xmm16[15],xmm14[15] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm11 = xmm11[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm11 = ymm11[0,1,0,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm11, %zmm26 -; AVX512BW-FAST-NEXT: vmovdqa (%rdi), %ymm11 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm28 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128] -; AVX512BW-FAST-NEXT: vpshufb %ymm28, %ymm11, %ymm6 -; AVX512BW-FAST-NEXT: vmovdqa (%rsi), %ymm12 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm29 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18] -; AVX512BW-FAST-NEXT: vpshufb %ymm29, %ymm12, %ymm18 -; AVX512BW-FAST-NEXT: vporq %ymm6, %ymm18, %ymm6 -; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %xmm18 -; AVX512BW-FAST-NEXT: vmovdqa64 (%rsi), %xmm19 -; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm27 = xmm19[8],xmm18[8],xmm19[9],xmm18[9],xmm19[10],xmm18[10],xmm19[11],xmm18[11],xmm19[12],xmm18[12],xmm19[13],xmm18[13],xmm19[14],xmm18[14],xmm19[15],xmm18[15] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm27 = xmm27[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm27 = ymm27[0,1,0,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm6, %zmm27, %zmm6 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm4, %zmm21 {%k1} +; AVX512BW-FAST-NEXT: vmovdqa (%rdx), %ymm2 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm19 = [0,1,0,1,14,128,14,15,0,1,14,15,128,13,14,15,16,17,16,128,30,31,30,31,16,17,128,31,28,29,30,31] +; AVX512BW-FAST-NEXT: vpshufb %ymm19, %ymm2, %ymm4 +; AVX512BW-FAST-NEXT: vmovdqa %ymm2, %ymm5 +; AVX512BW-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX512BW-FAST-NEXT: vmovdqa (%rcx), %ymm2 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm20 = [128,128,128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128] +; AVX512BW-FAST-NEXT: vpshufb %ymm20, %ymm2, %ymm9 +; AVX512BW-FAST-NEXT: vmovdqa64 %ymm2, %ymm25 +; AVX512BW-FAST-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill +; AVX512BW-FAST-NEXT: vpor %ymm4, %ymm9, %ymm4 +; AVX512BW-FAST-NEXT: vmovdqa (%rdx), %xmm13 +; AVX512BW-FAST-NEXT: vmovdqa (%rcx), %xmm15 +; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm9 = xmm15[8],xmm13[8],xmm15[9],xmm13[9],xmm15[10],xmm13[10],xmm15[11],xmm13[11],xmm15[12],xmm13[12],xmm15[13],xmm13[13],xmm15[14],xmm13[14],xmm15[15],xmm13[15] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm9 = xmm9[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm9 = ymm9[0,1,0,1] +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm9, %zmm22 +; AVX512BW-FAST-NEXT: vmovdqa (%rdi), %ymm14 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm23 = [0,1,14,128,12,13,0,1,14,15,128,3,12,13,2,3,16,128,30,31,28,29,16,17,128,31,18,19,28,29,18,128] +; AVX512BW-FAST-NEXT: vpshufb %ymm23, %ymm14, %ymm4 +; AVX512BW-FAST-NEXT: vmovdqa (%rsi), %ymm10 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm24 = [128,128,128,14,128,128,128,128,128,128,15,128,128,128,128,128,128,16,128,128,128,128,128,128,17,128,128,128,128,128,128,18] +; AVX512BW-FAST-NEXT: vpshufb %ymm24, %ymm10, %ymm17 +; AVX512BW-FAST-NEXT: vporq %ymm4, %ymm17, %ymm4 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rdi), %xmm17 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rsi), %xmm18 +; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm26 = xmm18[8],xmm17[8],xmm18[9],xmm17[9],xmm18[10],xmm17[10],xmm18[11],xmm17[11],xmm18[12],xmm17[12],xmm18[13],xmm17[13],xmm18[14],xmm17[14],xmm18[15],xmm17[15] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm26 = xmm26[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm26 = ymm26[0,1,0,1] +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm4, %zmm26, %zmm8 ; AVX512BW-FAST-NEXT: movabsq $435749860008887046, %r10 # imm = 0x60C183060C18306 ; AVX512BW-FAST-NEXT: kmovq %r10, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm26, %zmm6 {%k1} +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm22, %zmm8 {%k1} ; AVX512BW-FAST-NEXT: movabsq $4066998693416279096, %r10 # imm = 0x3870E1C3870E1C38 ; AVX512BW-FAST-NEXT: kmovq %r10, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm22, %zmm6 {%k1} -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm22 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10] -; AVX512BW-FAST-NEXT: vpermw %ymm13, %ymm22, %ymm22 -; AVX512BW-FAST-NEXT: vpshufb %ymm0, %ymm13, %ymm0 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm22, %zmm0, %zmm22 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%r9), %ymm27 -; AVX512BW-FAST-NEXT: vpshufb %ymm17, %ymm27, %ymm17 -; AVX512BW-FAST-NEXT: vmovdqa 32(%r8), %ymm1 -; AVX512BW-FAST-NEXT: vpshufb %ymm20, %ymm1, %ymm20 -; AVX512BW-FAST-NEXT: vporq %ymm17, %ymm20, %ymm17 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm20 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm1[20],zero,ymm1[18],zero,ymm1[20,21,20,21],zero,ymm1[19],zero,ymm1[19,20,21,22],zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm20 = ymm20[2,3,2,3] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm26 = ymm27[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm27[20],zero,ymm27[18],zero,zero,zero,zero,ymm27[21],zero,ymm27[19],zero,zero,zero,zero,ymm27[22] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm26 = ymm26[2,3,2,3] -; AVX512BW-FAST-NEXT: vporq %ymm20, %ymm26, %ymm20 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm20, %zmm17, %zmm26 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm21, %zmm8 {%k1} +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} ymm21 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10] +; AVX512BW-FAST-NEXT: vpermw %ymm11, %ymm21, %ymm21 +; AVX512BW-FAST-NEXT: vpshufb %ymm0, %ymm11, %ymm0 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm21, %zmm0, %zmm21 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%r9), %ymm28 +; AVX512BW-FAST-NEXT: vpshufb %ymm1, %ymm28, %ymm1 +; AVX512BW-FAST-NEXT: vmovdqa 32(%r8), %ymm0 +; AVX512BW-FAST-NEXT: vpshufb %ymm16, %ymm0, %ymm16 +; AVX512BW-FAST-NEXT: vporq %ymm1, %ymm16, %ymm1 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm2 = +; AVX512BW-FAST-NEXT: vpshufb %ymm2, %ymm28, %ymm16 +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm16 = ymm16[2,3,2,3] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm22 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm0[20],zero,ymm0[18],zero,ymm0[20,21,20,21],zero,ymm0[19],zero,ymm0[19,20,21,22],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm22 = ymm22[2,3,2,3] +; AVX512BW-FAST-NEXT: vporq %ymm22, %ymm16, %ymm16 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm16, %zmm1, %zmm1 ; AVX512BW-FAST-NEXT: movabsq $145249953336295682, %r10 # imm = 0x204081020408102 ; AVX512BW-FAST-NEXT: kmovq %r10, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm22, %zmm26 {%k1} -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %ymm22 -; AVX512BW-FAST-NEXT: vpshufb %ymm21, %ymm22, %ymm17 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rcx), %ymm30 -; AVX512BW-FAST-NEXT: vpshufb %ymm23, %ymm30, %ymm20 -; AVX512BW-FAST-NEXT: vporq %ymm17, %ymm20, %ymm17 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm20 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm22[18],zero,ymm22[18,19,20,21],zero,ymm22[19],zero,ymm22[25,26,27,22],zero,ymm22[20],zero +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm21, %zmm1 {%k1} +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %ymm26 +; AVX512BW-FAST-NEXT: vpshufb %ymm19, %ymm26, %ymm16 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rcx), %ymm29 +; AVX512BW-FAST-NEXT: vpshufb %ymm20, %ymm29, %ymm19 +; AVX512BW-FAST-NEXT: vporq %ymm16, %ymm19, %ymm16 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm2 = +; AVX512BW-FAST-NEXT: vpshufb %ymm2, %ymm29, %ymm20 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm20 = ymm20[2,3,2,3] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm21 = ymm30[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm30[18],zero,zero,zero,zero,ymm30[21],zero,ymm30[19],zero,zero,zero,zero,ymm30[22],zero,ymm30[20] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm21 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm26[18],zero,ymm26[18,19,20,21],zero,ymm26[19],zero,ymm26[25,26,27,22],zero,ymm26[20],zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm21 = ymm21[2,3,2,3] -; AVX512BW-FAST-NEXT: vporq %ymm20, %ymm21, %ymm20 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm20, %zmm17, %zmm21 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %ymm31 -; AVX512BW-FAST-NEXT: vpshufb %ymm28, %ymm31, %ymm17 -; AVX512BW-FAST-NEXT: vmovdqa 32(%rsi), %ymm0 -; AVX512BW-FAST-NEXT: vpshufb %ymm29, %ymm0, %ymm20 -; AVX512BW-FAST-NEXT: vporq %ymm17, %ymm20, %ymm17 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm20 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm31[18,19,20,21],zero,ymm31[19],zero,ymm31[21,20,21,22],zero,ymm31[20],zero,ymm31[22,23] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm20 = ymm20[2,3,2,3] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm23 = ymm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm0[21],zero,ymm0[19],zero,zero,zero,zero,ymm0[22],zero,ymm0[20],zero,zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm23 = ymm23[2,3,2,3] -; AVX512BW-FAST-NEXT: vporq %ymm20, %ymm23, %ymm20 -; AVX512BW-FAST-NEXT: vmovdqa64 (%rcx), %zmm23 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm20, %zmm17, %zmm17 -; AVX512BW-FAST-NEXT: vmovdqa64 (%r8), %zmm3 -; AVX512BW-FAST-NEXT: vmovdqu64 %zmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill +; AVX512BW-FAST-NEXT: vporq %ymm21, %ymm20, %ymm20 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm20, %zmm16, %zmm20 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %ymm30 +; AVX512BW-FAST-NEXT: vpshufb %ymm23, %ymm30, %ymm16 +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rsi), %ymm31 +; AVX512BW-FAST-NEXT: vpshufb %ymm24, %ymm31, %ymm21 +; AVX512BW-FAST-NEXT: vporq %ymm16, %ymm21, %ymm16 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm21 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,ymm30[18,19,20,21],zero,ymm30[19],zero,ymm30[21,20,21,22],zero,ymm30[20],zero,ymm30[22,23] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm21 = ymm21[2,3,2,3] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm22 = ymm31[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm31[21],zero,ymm31[19],zero,zero,zero,zero,ymm31[22],zero,ymm31[20],zero,zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm22 = ymm22[2,3,2,3] +; AVX512BW-FAST-NEXT: vporq %ymm21, %ymm22, %ymm21 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rsi), %zmm24 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm21, %zmm16, %zmm16 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rdx), %zmm22 ; AVX512BW-FAST-NEXT: movabsq $3485998880071096368, %r10 # imm = 0x3060C183060C1830 ; AVX512BW-FAST-NEXT: kmovq %r10, %k2 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm21, %zmm17 {%k2} -; AVX512BW-FAST-NEXT: vmovdqa64 (%r9), %zmm21 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm20, %zmm16 {%k2} +; AVX512BW-FAST-NEXT: vmovdqa64 (%rcx), %zmm23 ; AVX512BW-FAST-NEXT: movabsq $-4357498600088870461, %r10 # imm = 0xC3870E1C3870E1C3 ; AVX512BW-FAST-NEXT: kmovq %r10, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm26, %zmm17 {%k1} -; AVX512BW-FAST-NEXT: vmovdqa64 (%rax), %zmm26 -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm22 = zmm22[0,1,2,3],zmm23[4,5,6,7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm22 = zmm22[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,zmm22[23],zero,zmm22[21,22,23,26],zero,zmm22[24],zero,zmm22[28,29,26,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,59],zero,zero,zero,zero,zmm22[62],zero,zmm22[60],zero,zero,zero,zero,zmm22[63],zero,zmm22[61],zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm22 = zmm22[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm28 = zmm30[0,1,2,3],zmm2[4,5,6,7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm28 = zmm28[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm28[25],zero,zmm28[23],zero,zero,zero,zero,zmm28[26],zero,zmm28[24],zero,zero,zero,zero,zmm28[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm28[62],zero,zmm28[60],zero,zero,zero,zero,zmm28[63],zero,zmm28[61],zero,zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm28 = zmm28[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vporq %zmm22, %zmm28, %zmm29 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %xmm28 -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm9[4,5,6,7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,zmm0[23],zero,zero,zero,zero,zmm0[26],zero,zmm0[24],zero,zero,zero,zero,zmm0[27],zero,zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,60,61,62],zero,zmm0[60],zero,zmm0[62,63,62,63],zero,zmm0[61],zero,zmm0[63,60,61] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm22 = zmm31[0,1,2,3],zmm5[4,5,6,7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm22 = zmm22[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm22[23],zero,zero,zero,zero,zmm22[26],zero,zmm22[24],zero,zero,zero,zero,zmm22[27],zero,zmm22[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zmm22[62],zero,zmm22[60],zero,zero,zero,zero,zmm22[63],zero,zmm22[61],zero,zero,zero -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm22 = zmm22[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vporq %zmm0, %zmm22, %zmm22 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rsi), %xmm30 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm16 {%k1} +; AVX512BW-FAST-NEXT: vmovdqa64 (%r8), %zmm27 +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm26[0,1,2,3],zmm23[4,5,6,7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25],zero,zmm1[23],zero,zmm1[21,22,23,26],zero,zmm1[24],zero,zmm1[28,29,26,27,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,59],zero,zero,zero,zero,zmm1[62],zero,zmm1[60],zero,zero,zero,zero,zmm1[63],zero,zmm1[61],zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7] +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm29[0,1,2,3],zmm22[4,5,6,7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm20 = zmm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm20[25],zero,zmm20[23],zero,zero,zero,zero,zmm20[26],zero,zmm20[24],zero,zero,zero,zero,zmm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm20[62],zero,zmm20[60],zero,zero,zero,zero,zmm20[63],zero,zmm20[61],zero,zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm20 = zmm20[2,3,2,3,6,7,6,7] +; AVX512BW-FAST-NEXT: vporq %zmm1, %zmm20, %zmm1 +; AVX512BW-FAST-NEXT: vmovdqa64 (%r9), %zmm26 +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm20 = zmm31[0,1,2,3],zmm7[4,5,6,7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm20 = zmm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,25],zero,zmm20[23],zero,zero,zero,zero,zmm20[26],zero,zmm20[24],zero,zero,zero,zero,zmm20[27],zero,zmm20[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,60,61,62],zero,zmm20[60],zero,zmm20[62,63,62,63],zero,zmm20[61],zero,zmm20[63,60,61] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm20 = zmm20[2,3,2,3,6,7,6,7] +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm29 = zmm30[0,1,2,3],zmm24[4,5,6,7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm29 = zmm29[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm29[23],zero,zero,zero,zero,zmm29[26],zero,zmm29[24],zero,zero,zero,zero,zmm29[27],zero,zmm29[25,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zmm29[62],zero,zmm29[60],zero,zero,zero,zero,zmm29[63],zero,zmm29[61],zero,zero,zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm29 = zmm29[2,3,2,3,6,7,6,7] +; AVX512BW-FAST-NEXT: vporq %zmm20, %zmm29, %zmm20 +; AVX512BW-FAST-NEXT: vmovdqa64 (%rax), %zmm29 ; AVX512BW-FAST-NEXT: movabsq $1742999440035548184, %rax # imm = 0x183060C183060C18 ; AVX512BW-FAST-NEXT: kmovq %rax, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm29, %zmm22 {%k1} -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm21[4,5,6,7] +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm20 {%k1} +; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdi), %xmm31 +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm0[0,1,2,3],zmm26[4,5,6,7] ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm0 = zmm0[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,24,25,24,25],zero,zmm0[23],zero,zmm0[23,24,25,26],zero,zmm0[24],zero,zmm0[30,31,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,61],zero,zmm0[59],zero,zero,zero,zero,zmm0[62],zero,zmm0[60],zero,zero,zero,zero,zmm0[63],zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm0 = zmm0[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm27[0,1,2,3],zmm3[4,5,6,7] +; AVX512BW-FAST-NEXT: vshufi64x2 {{.*#+}} zmm1 = zmm28[0,1,2,3],zmm27[4,5,6,7] ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm1[25],zero,zmm1[23],zero,zero,zero,zero,zmm1[26],zero,zmm1[24],zero,zero,zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm1[59],zero,zero,zero,zero,zmm1[62],zero,zmm1[60],zero,zero,zero,zero,zmm1[63],zero,zmm1[61] ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7] ; AVX512BW-FAST-NEXT: vporq %zmm0, %zmm1, %zmm0 ; AVX512BW-FAST-NEXT: movabsq $6971997760142192736, %rax # imm = 0x60C183060C183060 ; AVX512BW-FAST-NEXT: kmovq %rax, %k1 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm0, %zmm22 {%k1} +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm0, %zmm20 {%k1} ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm0 = [11,13,12,11,12,13,13,12,11,13,12,11,12,13,13,12,62,61,62,63,63,62,62,63,62,61,62,63,63,62,62,63] -; AVX512BW-FAST-NEXT: vpermi2w %zmm26, %zmm13, %zmm0 +; AVX512BW-FAST-NEXT: vpermi2w %zmm29, %zmm11, %zmm0 ; AVX512BW-FAST-NEXT: movabsq $-9150747060186627967, %rax # imm = 0x8102040810204081 ; AVX512BW-FAST-NEXT: kmovq %rax, %k3 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm0, %zmm22 {%k3} -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm11[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,30],zero,ymm11[28],zero,ymm11[30,31,30,31],zero,ymm11[29],zero,ymm11[31,28,29] +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm0, %zmm20 {%k3} +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm14[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,28,29,30],zero,ymm14[28],zero,ymm14[30,31,30,31],zero,ymm14[29],zero,ymm14[31,28,29] ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm12[30],zero,ymm12[28],zero,zero,zero,zero,ymm12[31],zero,ymm12[29],zero,zero,zero +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm1 = ymm10[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,ymm10[30],zero,ymm10[28],zero,zero,zero,zero,ymm10[31],zero,ymm10[29],zero,zero,zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm1 = ymm1[2,3,2,3] ; AVX512BW-FAST-NEXT: vpor %ymm0, %ymm1, %ymm0 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm28[0],xmm30[0],xmm28[1],xmm30[1],xmm28[2],xmm30[2],xmm28[3],xmm30[3],xmm28[4],xmm30[4],xmm28[5],xmm30[5],xmm28[6],xmm30[6],xmm28[7],xmm30[7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,0,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm3 -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm8[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,zero,zero,ymm8[30],zero,ymm8[28],zero,zero,zero,zero,ymm8[31],zero,ymm8[29],zero +; AVX512BW-FAST-NEXT: vmovdqa 32(%rsi), %xmm7 +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm31[0],xmm7[0],xmm31[1],xmm7[1],xmm31[2],xmm7[2],xmm31[3],xmm7[3],xmm31[4],xmm7[4],xmm31[5],xmm7[5],xmm31[6],xmm7[6],xmm31[7],xmm7[7] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm2 = xmm2[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5] +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,1,0,1] +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm6 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm25[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,27],zero,zero,zero,zero,ymm25[30],zero,ymm25[28],zero,zero,zero,zero,ymm25[31],zero,ymm25[29],zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm27 = ymm7[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm7[30],zero,ymm7[28],zero,zero,zero,zero,ymm7[31],zero,ymm7[29],zero,zero -; AVX512BW-FAST-NEXT: vmovdqa64 %ymm7, %ymm20 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm27 = ymm27[2,3,2,3] -; AVX512BW-FAST-NEXT: vporq %ymm0, %ymm27, %ymm27 -; AVX512BW-FAST-NEXT: vmovdqa64 32(%rdx), %xmm31 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm28 = ymm5[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,ymm5[30],zero,ymm5[28],zero,zero,zero,zero,ymm5[31],zero,ymm5[29],zero,zero +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm28 = ymm28[2,3,2,3] +; AVX512BW-FAST-NEXT: vporq %ymm0, %ymm28, %ymm28 +; AVX512BW-FAST-NEXT: vmovdqa 32(%rdx), %xmm2 ; AVX512BW-FAST-NEXT: vmovdqa 32(%rcx), %xmm1 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm31[0],xmm1[0],xmm31[1],xmm1[1],xmm31[2],xmm1[2],xmm31[3],xmm1[3],xmm31[4],xmm1[4],xmm31[5],xmm1[5],xmm31[6],xmm1[6],xmm31[7],xmm1[7] -; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm7 = <4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9> -; AVX512BW-FAST-NEXT: vpshufb %xmm7, %xmm0, %xmm0 +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm9 = <4,5,0,1,u,u,u,6,7,2,3,u,u,u,8,9> +; AVX512BW-FAST-NEXT: vpshufb %xmm9, %xmm0, %xmm0 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm27, %zmm27 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm3, %zmm27 {%k2} -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm15[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm15[27],zero,zero,zero,zero,ymm15[30],zero,ymm15[28],zero,zero,zero,zero,ymm15[31],zero +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm28, %zmm28 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm6, %zmm28 {%k2} +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm0 = ymm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,29],zero,ymm3[27],zero,zero,zero,zero,ymm3[30],zero,ymm3[28],zero,zero,zero,zero,ymm3[31],zero +; AVX512BW-FAST-NEXT: vmovdqa64 %ymm3, %ymm30 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[2,3,2,3] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm3 = ymm24[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm24[27],zero,zero,zero,zero,ymm24[30],zero,ymm24[28],zero,zero,zero,zero,ymm24[31],zero,ymm24[29] -; AVX512BW-FAST-NEXT: vmovdqa64 %ymm24, %ymm9 -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm3 = ymm3[2,3,2,3] -; AVX512BW-FAST-NEXT: vpor %ymm0, %ymm3, %ymm2 -; AVX512BW-FAST-NEXT: vmovdqa 32(%r9), %xmm4 -; AVX512BW-FAST-NEXT: vmovdqa 32(%r8), %xmm3 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3],xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7] -; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm8 = -; AVX512BW-FAST-NEXT: vpshufb %xmm8, %xmm0, %xmm0 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} ymm6 = ymm12[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,ymm12[27],zero,zero,zero,zero,ymm12[30],zero,ymm12[28],zero,zero,zero,zero,ymm12[31],zero,ymm12[29] +; AVX512BW-FAST-NEXT: vmovdqa64 %ymm12, %ymm21 +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm6 = ymm6[2,3,2,3] +; AVX512BW-FAST-NEXT: vpor %ymm0, %ymm6, %ymm3 +; AVX512BW-FAST-NEXT: vmovdqa 32(%r9), %xmm5 +; AVX512BW-FAST-NEXT: vmovdqa 32(%r8), %xmm4 +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm4[0],xmm5[0],xmm4[1],xmm5[1],xmm4[2],xmm5[2],xmm4[3],xmm5[3],xmm4[4],xmm5[4],xmm4[5],xmm5[5],xmm4[6],xmm5[6],xmm4[7],xmm5[7] +; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm12 = +; AVX512BW-FAST-NEXT: vpshufb %xmm12, %xmm0, %xmm0 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,0,1] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm2 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16] -; AVX512BW-FAST-NEXT: vpermw %zmm26, %zmm2, %zmm2 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm0, %zmm3, %zmm0 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm3 = [14,13,14,15,15,14,14,15,14,13,14,15,15,14,14,15,17,17,16,16,17,17,16,16,20,21,17,17,17,17,16,16] +; AVX512BW-FAST-NEXT: vpermw %zmm29, %zmm3, %zmm3 ; AVX512BW-FAST-NEXT: movabsq $580999813345182728, %rax # imm = 0x810204081020408 ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm2, %zmm0 {%k2} +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm3, %zmm0 {%k2} ; AVX512BW-FAST-NEXT: movabsq $1016749673354069774, %rax # imm = 0xE1C3870E1C3870E ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm0, %zmm27 {%k2} -; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm5 = -; AVX512BW-FAST-NEXT: vpshufb %xmm5, %xmm1, %xmm2 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm0, %zmm28 {%k2} +; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm6 = +; AVX512BW-FAST-NEXT: vpshufb %xmm6, %xmm1, %xmm3 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm0 = -; AVX512BW-FAST-NEXT: vpshufb %xmm0, %xmm31, %xmm24 -; AVX512BW-FAST-NEXT: vporq %xmm2, %xmm24, %xmm2 -; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm31[8],xmm1[9],xmm31[9],xmm1[10],xmm31[10],xmm1[11],xmm31[11],xmm1[12],xmm31[12],xmm1[13],xmm31[13],xmm1[14],xmm31[14],xmm1[15],xmm31[15] +; AVX512BW-FAST-NEXT: vpshufb %xmm0, %xmm2, %xmm19 +; AVX512BW-FAST-NEXT: vporq %xmm3, %xmm19, %xmm3 +; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15] ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[6,3,2,u,u,u,9,8,5,4,u,u,u,11,10,7] -; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm1 +; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm3, %zmm1 ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm2 = -; AVX512BW-FAST-NEXT: vpshufb %xmm2, %xmm30, %xmm24 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm31 = -; AVX512BW-FAST-NEXT: vpshufb %xmm31, %xmm28, %xmm29 -; AVX512BW-FAST-NEXT: vporq %xmm24, %xmm29, %xmm24 -; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm28 = xmm30[8],xmm28[8],xmm30[9],xmm28[9],xmm30[10],xmm28[10],xmm30[11],xmm28[11],xmm30[12],xmm28[12],xmm30[13],xmm28[13],xmm30[14],xmm28[14],xmm30[15],xmm28[15] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm28 = xmm28[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u] -; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm28, %zmm24, %zmm24 +; AVX512BW-FAST-NEXT: vpshufb %xmm2, %xmm7, %xmm3 +; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm19 = +; AVX512BW-FAST-NEXT: vpshufb %xmm19, %xmm31, %xmm25 +; AVX512BW-FAST-NEXT: vporq %xmm3, %xmm25, %xmm3 +; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm7 = xmm7[8],xmm31[8],xmm7[9],xmm31[9],xmm7[10],xmm31[10],xmm7[11],xmm31[11],xmm7[12],xmm31[12],xmm7[13],xmm31[13],xmm7[14],xmm31[14],xmm7[15],xmm31[15] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm7 = xmm7[2,u,u,u,9,8,5,4,u,u,u,11,10,7,6,u] +; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm7, %zmm3, %zmm3 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,1,0,1,4,5,4,5] -; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm24 = zmm24[0,1,0,1,4,5,4,5] -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm24 {%k1} +; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm25 = zmm3[0,1,0,1,4,5,4,5] +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm25 {%k1} ; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm1 = <128,4,u,u,u,128,7,128,5,u,u,u,128,8,128,6> -; AVX512BW-FAST-NEXT: vpshufb %xmm1, %xmm4, %xmm28 -; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} xmm29 = <4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128> -; AVX512BW-FAST-NEXT: vpshufb %xmm29, %xmm3, %xmm30 -; AVX512BW-FAST-NEXT: vporq %xmm28, %xmm30, %xmm28 -; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10] -; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm3, %zmm28, %zmm3 +; AVX512BW-FAST-NEXT: vpshufb %xmm1, %xmm5, %xmm3 +; AVX512BW-FAST-NEXT: vmovdqa {{.*#+}} xmm7 = <4,128,u,u,u,7,128,5,128,u,u,u,8,128,6,128> +; AVX512BW-FAST-NEXT: vpshufb %xmm7, %xmm4, %xmm31 +; AVX512BW-FAST-NEXT: vporq %xmm3, %xmm31, %xmm3 +; AVX512BW-FAST-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm4[8],xmm5[8],xmm4[9],xmm5[9],xmm4[10],xmm5[10],xmm4[11],xmm5[11],xmm4[12],xmm5[12],xmm4[13],xmm5[13],xmm4[14],xmm5[14],xmm4[15],xmm5[15] +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm4 = xmm4[u,6,7,2,3,u,u,u,8,9,4,5,u,u,u,10] +; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm4, %zmm3, %zmm3 ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [2,2,2,4,2,2,2,4,3,3,3,3,2,2,2,4,52,53,52,53,53,54,53,54,52,53,52,53,53,54,53,54] -; AVX512BW-FAST-NEXT: vpermi2w %zmm26, %zmm13, %zmm4 +; AVX512BW-FAST-NEXT: vpermi2w %zmm29, %zmm11, %zmm4 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm3[0,1,0,1,4,5,4,5] ; AVX512BW-FAST-NEXT: movabsq $290499906672591364, %rax # imm = 0x408102040810204 ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm4, %zmm3 {%k2} ; AVX512BW-FAST-NEXT: movabsq $-8714997200177740921, %rax # imm = 0x870E1C3870E1C387 ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 -; AVX512BW-FAST-NEXT: vmovdqu8 %zmm3, %zmm24 {%k2} -; AVX512BW-FAST-NEXT: vpshufb %xmm5, %xmm16, %xmm3 -; AVX512BW-FAST-NEXT: vpshufb %xmm0, %xmm14, %xmm0 +; AVX512BW-FAST-NEXT: vmovdqu8 %zmm3, %zmm25 {%k2} +; AVX512BW-FAST-NEXT: vpshufb %xmm6, %xmm15, %xmm3 +; AVX512BW-FAST-NEXT: vpshufb %xmm0, %xmm13, %xmm0 ; AVX512BW-FAST-NEXT: vpor %xmm3, %xmm0, %xmm0 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm14[0],xmm16[0],xmm14[1],xmm16[1],xmm14[2],xmm16[2],xmm14[3],xmm16[3],xmm14[4],xmm16[4],xmm14[5],xmm16[5],xmm14[6],xmm16[6],xmm14[7],xmm16[7] -; AVX512BW-FAST-NEXT: vpshufb %xmm7, %xmm3, %xmm3 +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm13[0],xmm15[0],xmm13[1],xmm15[1],xmm13[2],xmm15[2],xmm13[3],xmm15[3],xmm13[4],xmm15[4],xmm13[5],xmm15[5],xmm13[6],xmm15[6],xmm13[7],xmm15[7] +; AVX512BW-FAST-NEXT: vpshufb %xmm9, %xmm3, %xmm3 ; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm3, %zmm0 -; AVX512BW-FAST-NEXT: vpshufb %xmm2, %xmm19, %xmm2 -; AVX512BW-FAST-NEXT: vpshufb %xmm31, %xmm18, %xmm3 +; AVX512BW-FAST-NEXT: vpshufb %xmm2, %xmm18, %xmm2 +; AVX512BW-FAST-NEXT: vpshufb %xmm19, %xmm17, %xmm3 ; AVX512BW-FAST-NEXT: vpor %xmm2, %xmm3, %xmm2 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm18[0],xmm19[0],xmm18[1],xmm19[1],xmm18[2],xmm19[2],xmm18[3],xmm19[3],xmm18[4],xmm19[4],xmm18[5],xmm19[5],xmm18[6],xmm19[6],xmm18[7],xmm19[7] +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm3 = xmm17[0],xmm18[0],xmm17[1],xmm18[1],xmm17[2],xmm18[2],xmm17[3],xmm18[3],xmm17[4],xmm18[4],xmm17[5],xmm18[5],xmm17[6],xmm18[6],xmm17[7],xmm18[7] ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm3 = xmm3[0,1,u,u,u,6,7,2,3,u,u,u,8,9,4,5] ; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm2, %zmm3, %zmm2 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm0[0,1,0,1,4,5,4,5] @@ -8666,14 +8667,16 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-NEXT: movabsq $871499720017774092, %rax # imm = 0xC183060C183060C ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm3, %zmm0 {%k2} -; AVX512BW-FAST-NEXT: vpshufb %xmm1, %xmm25, %xmm1 -; AVX512BW-FAST-NEXT: vpshufb %xmm29, %xmm10, %xmm2 +; AVX512BW-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload +; AVX512BW-FAST-NEXT: vpshufb %xmm1, %xmm3, %xmm1 +; AVX512BW-FAST-NEXT: vmovdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload +; AVX512BW-FAST-NEXT: vpshufb %xmm7, %xmm4, %xmm2 ; AVX512BW-FAST-NEXT: vpor %xmm1, %xmm2, %xmm1 -; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm10[0],xmm25[0],xmm10[1],xmm25[1],xmm10[2],xmm25[2],xmm10[3],xmm25[3],xmm10[4],xmm25[4],xmm10[5],xmm25[5],xmm10[6],xmm25[6],xmm10[7],xmm25[7] -; AVX512BW-FAST-NEXT: vpshufb %xmm8, %xmm2, %xmm2 +; AVX512BW-FAST-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7] +; AVX512BW-FAST-NEXT: vpshufb %xmm12, %xmm2, %xmm2 ; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm1, %zmm2, %zmm1 ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[0,1,0,1,4,5,4,5] -; AVX512BW-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm26, %zmm2 # 32-byte Folded Reload +; AVX512BW-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm29, %zmm2 # 32-byte Folded Reload ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm3 = [1,1,0,0,1,1,0,0,4,5,1,1,1,1,0,0,18,18,18,20,18,18,18,20,19,19,19,19,18,18,18,20] ; AVX512BW-FAST-NEXT: vpermw %zmm2, %zmm3, %zmm3 ; AVX512BW-FAST-NEXT: movabsq $4647998506761461824, %rax # imm = 0x4081020408102040 @@ -8683,29 +8686,26 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-NEXT: kmovq %rax, %k2 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm0 {%k2} ; AVX512BW-FAST-NEXT: vmovdqu64 (%rsp), %zmm1 # 64-byte Reload -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm12, %zmm1, %zmm1 -; AVX512BW-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm11, %zmm3, %zmm3 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm10, %zmm1, %zmm1 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm14, %zmm24, %zmm3 ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18,19,20,21],zero,zmm1[19],zero,zmm1[21,20,21,22],zero,zmm1[20],zero,zmm1[22,23,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,57],zero,zmm1[55],zero,zero,zero,zero,zmm1[58],zero,zmm1[56],zero,zero,zero,zero,zmm1[59],zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7] ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm3[21],zero,zmm3[19],zero,zero,zero,zero,zmm3[22],zero,zmm3[20],zero,zero,zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm3[55],zero,zero,zero,zero,zmm3[58],zero,zmm3[56],zero,zero,zero,zero,zmm3[59],zero,zmm3[57] ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm3[2,3,2,3,6,7,6,7] ; AVX512BW-FAST-NEXT: vporq %zmm1, %zmm3, %zmm1 -; AVX512BW-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm3 # 64-byte Reload -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm20, %zmm3, %zmm3 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm22, %zmm3 # 32-byte Folded Reload ; AVX512BW-FAST-NEXT: vinserti64x4 $1, {{[-0-9]+}}(%r{{[sb]}}p), %zmm23, %zmm4 # 32-byte Folded Reload +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm4[18],zero,zero,zero,zero,zmm4[21],zero,zmm4[19],zero,zero,zero,zero,zmm4[22],zero,zmm4[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm4[57],zero,zmm4[55],zero,zero,zero,zero,zmm4[58],zero,zmm4[56],zero,zero,zero,zero ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm3 = zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,18],zero,zmm3[18,19,20,21],zero,zmm3[19],zero,zmm3[25,26,27,22],zero,zmm3[20],zero,zmm3[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,56,57],zero,zmm3[55],zero,zmm3[53,54,55,58],zero,zmm3[56],zero,zmm3[60,61,58,59] ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm3 = zmm3[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm4[18],zero,zero,zero,zero,zmm4[21],zero,zmm4[19],zero,zero,zero,zero,zmm4[22],zero,zmm4[20,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zmm4[57],zero,zmm4[55],zero,zero,zero,zero,zmm4[58],zero,zmm4[56],zero,zero,zero,zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm4 = zmm4[2,3,2,3,6,7,6,7] ; AVX512BW-FAST-NEXT: vporq %zmm3, %zmm4, %zmm3 ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm3 {%k1} -; AVX512BW-FAST-NEXT: vmovdqu64 {{[-0-9]+}}(%r{{[sb]}}p), %zmm1 # 64-byte Reload -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm9, %zmm1, %zmm1 -; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm15, %zmm21, %zmm4 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm21, %zmm27, %zmm1 +; AVX512BW-FAST-NEXT: vinserti64x4 $1, %ymm30, %zmm26, %zmm4 +; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm4[20],zero,zmm4[18],zero,zero,zero,zero,zmm4[21],zero,zmm4[19],zero,zero,zero,zero,zmm4[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm4[57],zero,zmm4[55],zero,zero,zero,zero,zmm4[58],zero,zmm4[56],zero,zero ; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm1 = zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,20],zero,zmm1[18],zero,zmm1[20,21,20,21],zero,zmm1[19],zero,zmm1[19,20,21,22],zero,zmm1[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,56,57,56,57],zero,zmm1[55],zero,zmm1[55,56,57,58],zero,zmm1[56],zero,zmm1[62,63] ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm1 = zmm1[2,3,2,3,6,7,6,7] -; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} zmm4 = zmm4[u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zmm4[20],zero,zmm4[18],zero,zero,zero,zero,zmm4[21],zero,zmm4[19],zero,zero,zero,zero,zmm4[22,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u],zero,zero,zero,zero,zmm4[57],zero,zmm4[55],zero,zero,zero,zero,zmm4[58],zero,zmm4[56],zero,zero ; AVX512BW-FAST-NEXT: vpermq {{.*#+}} zmm4 = zmm4[2,3,2,3,6,7,6,7] ; AVX512BW-FAST-NEXT: vporq %zmm1, %zmm4, %zmm1 ; AVX512BW-FAST-NEXT: vmovdqa64 {{.*#+}} zmm4 = [10,9,9,10,10,9,9,10,9,10,14,15,10,9,9,10,27,29,28,27,28,29,29,28,27,29,28,27,28,29,29,28] @@ -8718,13 +8718,13 @@ define void @store_i8_stride7_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.vec ; AVX512BW-FAST-NEXT: vmovdqu8 %zmm1, %zmm3 {%k1} ; AVX512BW-FAST-NEXT: movq {{[0-9]+}}(%rsp), %rax ; AVX512BW-FAST-NEXT: vmovdqa64 %zmm3, 128(%rax) -; AVX512BW-FAST-NEXT: vmovdqa64 %zmm17, 320(%rax) +; AVX512BW-FAST-NEXT: vmovdqa64 %zmm16, 320(%rax) ; AVX512BW-FAST-NEXT: vmovdqa64 %zmm0, (%rax) -; AVX512BW-FAST-NEXT: vmovdqa64 %zmm24, 256(%rax) -; AVX512BW-FAST-NEXT: vmovdqa64 %zmm27, 192(%rax) -; AVX512BW-FAST-NEXT: vmovdqa64 %zmm6, 64(%rax) -; AVX512BW-FAST-NEXT: vmovdqa64 %zmm22, 384(%rax) -; AVX512BW-FAST-NEXT: addq $200, %rsp +; AVX512BW-FAST-NEXT: vmovdqa64 %zmm25, 256(%rax) +; AVX512BW-FAST-NEXT: vmovdqa64 %zmm28, 192(%rax) +; AVX512BW-FAST-NEXT: vmovdqa64 %zmm8, 64(%rax) +; AVX512BW-FAST-NEXT: vmovdqa64 %zmm20, 384(%rax) +; AVX512BW-FAST-NEXT: addq $72, %rsp ; AVX512BW-FAST-NEXT: vzeroupper ; AVX512BW-FAST-NEXT: retq %in.vec0 = load <64 x i8>, ptr %in.vecptr0, align 64 diff --git a/llvm/test/CodeGen/X86/vector-rotate-128.ll b/llvm/test/CodeGen/X86/vector-rotate-128.ll index 33f7a4e42b7f..43c9be2dc6f9 100644 --- a/llvm/test/CodeGen/X86/vector-rotate-128.ll +++ b/llvm/test/CodeGen/X86/vector-rotate-128.ll @@ -1573,7 +1573,7 @@ define <16 x i8> @splatconstant_rotate_v16i8(<16 x i8> %a) nounwind { ; AVX512NOVLX: # %bb.0: ; AVX512NOVLX-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512NOVLX-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512NOVLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512NOVLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512NOVLX-NEXT: # kill: def $xmm0 killed $xmm0 killed $zmm0 ; AVX512NOVLX-NEXT: vzeroupper ; AVX512NOVLX-NEXT: retq @@ -1582,7 +1582,7 @@ define <16 x i8> @splatconstant_rotate_v16i8(<16 x i8> %a) nounwind { ; AVX512VLX: # %bb.0: ; AVX512VLX-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VLX-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VLX-NEXT: retq ; ; XOP-LABEL: splatconstant_rotate_v16i8: @@ -1811,7 +1811,7 @@ define <16 x i8> @splatconstant_rotate_mask_v16i8(<16 x i8> %a) nounwind { ; AVX512NOVLX: # %bb.0: ; AVX512NOVLX-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512NOVLX-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512NOVLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512NOVLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512NOVLX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 ; AVX512NOVLX-NEXT: vzeroupper ; AVX512NOVLX-NEXT: retq @@ -1820,7 +1820,7 @@ define <16 x i8> @splatconstant_rotate_mask_v16i8(<16 x i8> %a) nounwind { ; AVX512VLX: # %bb.0: ; AVX512VLX-NEXT: vpsllw $4, %xmm0, %xmm1 ; AVX512VLX-NEXT: vpsrlw $4, %xmm0, %xmm0 -; AVX512VLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm0 +; AVX512VLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm1, %xmm0 ; AVX512VLX-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0 ; AVX512VLX-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll index 7047d5e3131a..c55335f84956 100644 --- a/llvm/test/CodeGen/X86/vector-rotate-256.ll +++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll @@ -392,12 +392,12 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind { ; AVX512F: # %bb.0: ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: vpsllw $2, %ymm0, %ymm2 ; AVX512F-NEXT: vpsrlw $6, %ymm0, %ymm3 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm2, %zmm3 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm3 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm2 @@ -412,12 +412,12 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind { ; AVX512VL: # %bb.0: ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsllw $2, %ymm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $6, %ymm0, %ymm3 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm2, %ymm3 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm2, %ymm3 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm3, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $7, %ymm0, %ymm2 @@ -1404,7 +1404,7 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind { ; AVX512NOVLX: # %bb.0: ; AVX512NOVLX-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512NOVLX-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512NOVLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512NOVLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512NOVLX-NEXT: # kill: def $ymm0 killed $ymm0 killed $zmm0 ; AVX512NOVLX-NEXT: retq ; @@ -1412,7 +1412,7 @@ define <32 x i8> @splatconstant_rotate_v32i8(<32 x i8> %a) nounwind { ; AVX512VLX: # %bb.0: ; AVX512VLX-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VLX-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VLX-NEXT: retq ; ; XOPAVX1-LABEL: splatconstant_rotate_v32i8: @@ -1667,7 +1667,7 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind { ; AVX512NOVLX: # %bb.0: ; AVX512NOVLX-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512NOVLX-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512NOVLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512NOVLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512NOVLX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0 ; AVX512NOVLX-NEXT: retq ; @@ -1675,7 +1675,7 @@ define <32 x i8> @splatconstant_rotate_mask_v32i8(<32 x i8> %a) nounwind { ; AVX512VLX: # %bb.0: ; AVX512VLX-NEXT: vpsllw $4, %ymm0, %ymm1 ; AVX512VLX-NEXT: vpsrlw $4, %ymm0, %ymm0 -; AVX512VLX-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm1, %ymm0 +; AVX512VLX-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm0 ; AVX512VLX-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0 ; AVX512VLX-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-rotate-512.ll b/llvm/test/CodeGen/X86/vector-rotate-512.ll index 6504c3e6353e..29afbf4c62ef 100644 --- a/llvm/test/CodeGen/X86/vector-rotate-512.ll +++ b/llvm/test/CodeGen/X86/vector-rotate-512.ll @@ -135,15 +135,15 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { ; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm3 ; AVX512F-NEXT: vpsllw $4, %ymm2, %ymm4 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm5 = [17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520] -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm5, %zmm4 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm5 = [4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160] +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm5, %zmm4 ; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3 ; AVX512F-NEXT: vpsllw $5, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512F-NEXT: vpsrlw $6, %ymm2, %ymm4 ; AVX512F-NEXT: vpsllw $2, %ymm2, %ymm6 -; AVX512F-NEXT: vpbroadcastq {{.*#+}} zmm7 = [18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596] -; AVX512F-NEXT: vpternlogq $226, %zmm4, %zmm7, %zmm6 +; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm7 = [4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268] +; AVX512F-NEXT: vpternlogd $226, %zmm4, %zmm7, %zmm6 ; AVX512F-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512F-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512F-NEXT: vpsrlw $7, %ymm2, %ymm4 @@ -155,12 +155,12 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { ; AVX512F-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm3 ; AVX512F-NEXT: vpsllw $4, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm5, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm5, %zmm4 ; AVX512F-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $6, %ymm0, %ymm3 ; AVX512F-NEXT: vpsllw $2, %ymm0, %ymm4 -; AVX512F-NEXT: vpternlogq $226, %zmm3, %zmm7, %zmm4 +; AVX512F-NEXT: vpternlogd $226, %zmm3, %zmm7, %zmm4 ; AVX512F-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512F-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $7, %ymm0, %ymm3 @@ -177,15 +177,15 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { ; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm3 ; AVX512VL-NEXT: vpsllw $4, %ymm2, %ymm4 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm5 = [17361641481138401520,17361641481138401520,17361641481138401520,17361641481138401520] -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm5, %ymm4 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm5 = [4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160,4042322160] +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm5, %ymm4 ; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm3 ; AVX512VL-NEXT: vpsllw $5, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm4, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsrlw $6, %ymm2, %ymm4 ; AVX512VL-NEXT: vpsllw $2, %ymm2, %ymm6 -; AVX512VL-NEXT: vpbroadcastq {{.*#+}} ymm7 = [18229723555195321596,18229723555195321596,18229723555195321596,18229723555195321596] -; AVX512VL-NEXT: vpternlogq $226, %ymm4, %ymm7, %ymm6 +; AVX512VL-NEXT: vpbroadcastd {{.*#+}} ymm7 = [4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268,4244438268] +; AVX512VL-NEXT: vpternlogd $226, %ymm4, %ymm7, %ymm6 ; AVX512VL-NEXT: vpaddb %ymm3, %ymm3, %ymm3 ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsrlw $7, %ymm2, %ymm4 @@ -196,12 +196,12 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind { ; AVX512VL-NEXT: vpblendvb %ymm3, %ymm6, %ymm2, %ymm2 ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsllw $4, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm5, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm5, %ymm4 ; AVX512VL-NEXT: vpsllw $5, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $6, %ymm0, %ymm3 ; AVX512VL-NEXT: vpsllw $2, %ymm0, %ymm4 -; AVX512VL-NEXT: vpternlogq $226, %ymm3, %ymm7, %ymm4 +; AVX512VL-NEXT: vpternlogd $226, %ymm3, %ymm7, %ymm4 ; AVX512VL-NEXT: vpaddb %ymm1, %ymm1, %ymm1 ; AVX512VL-NEXT: vpblendvb %ymm1, %ymm4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $7, %ymm0, %ymm3 @@ -754,7 +754,7 @@ define <64 x i8> @splatconstant_rotate_v64i8(<64 x i8> %a) nounwind { ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512VL-LABEL: splatconstant_rotate_v64i8: @@ -766,35 +766,35 @@ define <64 x i8> @splatconstant_rotate_v64i8(<64 x i8> %a) nounwind { ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VL-NEXT: retq ; ; AVX512BW-LABEL: splatconstant_rotate_v64i8: ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: retq ; ; AVX512VLBW-LABEL: splatconstant_rotate_v64i8: ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLBW-NEXT: retq ; ; AVX512VBMI2-LABEL: splatconstant_rotate_v64i8: ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: retq ; ; AVX512VLVBMI2-LABEL: splatconstant_rotate_v64i8: ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLVBMI2-NEXT: retq %shl = shl <64 x i8> %a, %lshr = lshr <64 x i8> %a, @@ -902,7 +902,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind { ; AVX512F-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512F-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512F-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0 ; AVX512F-NEXT: retq ; @@ -915,7 +915,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind { ; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0 ; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm2 ; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0 -; AVX512VL-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VL-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0 ; AVX512VL-NEXT: retq ; @@ -923,7 +923,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind { ; AVX512BW: # %bb.0: ; AVX512BW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512BW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512BW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512BW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0 ; AVX512BW-NEXT: retq ; @@ -931,7 +931,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind { ; AVX512VLBW: # %bb.0: ; AVX512VLBW-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLBW-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLBW-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLBW-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0 ; AVX512VLBW-NEXT: retq ; @@ -939,7 +939,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind { ; AVX512VBMI2: # %bb.0: ; AVX512VBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0 ; AVX512VBMI2-NEXT: retq ; @@ -947,7 +947,7 @@ define <64 x i8> @splatconstant_rotate_mask_v64i8(<64 x i8> %a) nounwind { ; AVX512VLVBMI2: # %bb.0: ; AVX512VLVBMI2-NEXT: vpsllw $4, %zmm0, %zmm1 ; AVX512VLVBMI2-NEXT: vpsrlw $4, %zmm0, %zmm0 -; AVX512VLVBMI2-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm0 +; AVX512VLVBMI2-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm0 ; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0 ; AVX512VLVBMI2-NEXT: retq %shl = shl <64 x i8> %a, diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll index 938fba0490b5..e6b5998d965f 100644 --- a/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll +++ b/llvm/test/CodeGen/X86/vector-shift-ashr-128.ll @@ -1160,9 +1160,9 @@ define <16 x i8> @splatvar_modulo_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwi ; ; XOPAVX1-LABEL: splatvar_modulo_shift_v16i8: ; XOPAVX1: # %bb.0: -; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 ; XOPAVX1-NEXT: vpshab %xmm1, %xmm0, %xmm0 ; XOPAVX1-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll index 5a70e5d4a2b5..6d19a81d9fd7 100644 --- a/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll +++ b/llvm/test/CodeGen/X86/vector-shift-ashr-256.ll @@ -1265,9 +1265,9 @@ define <32 x i8> @splatvar_modulo_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwi ; ; XOPAVX1-LABEL: splatvar_modulo_shift_v32i8: ; XOPAVX1: # %bb.0: -; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 ; XOPAVX1-NEXT: vpshab %xmm1, %xmm2, %xmm2 diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll index e248aafab525..7a2dcd1c8ca8 100644 --- a/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll +++ b/llvm/test/CodeGen/X86/vector-shift-lshr-128.ll @@ -941,9 +941,9 @@ define <16 x i8> @splatvar_modulo_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwi ; ; XOPAVX1-LABEL: splatvar_modulo_shift_v16i8: ; XOPAVX1: # %bb.0: -; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 ; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 ; XOPAVX1-NEXT: retq diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll index 77f5f2660af7..5fe661c7e777 100644 --- a/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll +++ b/llvm/test/CodeGen/X86/vector-shift-lshr-256.ll @@ -1037,9 +1037,9 @@ define <32 x i8> @splatvar_modulo_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwi ; ; XOPAVX1-LABEL: splatvar_modulo_shift_v32i8: ; XOPAVX1: # %bb.0: -; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpsubb %xmm1, %xmm2, %xmm1 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 ; XOPAVX1-NEXT: vpshlb %xmm1, %xmm2, %xmm2 diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll index deb1514e42c4..12f971fb83b5 100644 --- a/llvm/test/CodeGen/X86/vector-shift-shl-128.ll +++ b/llvm/test/CodeGen/X86/vector-shift-shl-128.ll @@ -848,9 +848,9 @@ define <16 x i8> @splatvar_modulo_shift_v16i8(<16 x i8> %a, <16 x i8> %b) nounwi ; ; XOPAVX1-LABEL: splatvar_modulo_shift_v16i8: ; XOPAVX1: # %bb.0: -; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 ; XOPAVX1-NEXT: retq ; diff --git a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll index c355eeaa42b6..76944994c87d 100644 --- a/llvm/test/CodeGen/X86/vector-shift-shl-256.ll +++ b/llvm/test/CodeGen/X86/vector-shift-shl-256.ll @@ -962,9 +962,9 @@ define <32 x i8> @splatvar_modulo_shift_v32i8(<32 x i8> %a, <32 x i8> %b) nounwi ; ; XOPAVX1-LABEL: splatvar_modulo_shift_v32i8: ; XOPAVX1: # %bb.0: -; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshufb %xmm2, %xmm1, %xmm1 +; XOPAVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 ; XOPAVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 ; XOPAVX1-NEXT: vpshlb %xmm1, %xmm2, %xmm2 ; XOPAVX1-NEXT: vpshlb %xmm1, %xmm0, %xmm0 diff --git a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll index 1b6d2a2c6298..be4253b6d5d1 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-combining.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-combining.ll @@ -3225,7 +3225,7 @@ define void @PR43024() { ; AVX-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}+4(%rip), %xmm0, %xmm0 ; AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1 ; AVX-NEXT: vaddss %xmm1, %xmm0, %xmm0 -; AVX-NEXT: vaddss {{\.?LCPI[0-9]+_[0-9]+}}+12(%rip), %xmm0, %xmm0 +; AVX-NEXT: vaddss %xmm1, %xmm0, %xmm0 ; AVX-NEXT: vmovss %xmm0, (%rax) ; AVX-NEXT: retq store <4 x float> , ptr undef, align 16 diff --git a/llvm/test/CodeGen/X86/vector-shuffle-v192.ll b/llvm/test/CodeGen/X86/vector-shuffle-v192.ll index 7159edc2bbdf..75c26f38eb9e 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-v192.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-v192.ll @@ -214,23 +214,21 @@ define <64 x i8> @f2(ptr %p0) { ; AVX512F-NEXT: vmovdqa 128(%rdi), %ymm4 ; AVX512F-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,3,5,9,11,15,17,21,23,27,29],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero ; AVX512F-NEXT: vinserti64x4 $1, %ymm4, %zmm2, %zmm2 -; AVX512F-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255] -; AVX512F-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3] -; AVX512F-NEXT: vpternlogq $234, %zmm2, %zmm0, %zmm4 +; AVX512F-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2 ; AVX512F-NEXT: vmovdqa 96(%rdi), %xmm0 ; AVX512F-NEXT: vpshufb %xmm5, %xmm0, %xmm0 -; AVX512F-NEXT: vmovdqa 112(%rdi), %xmm2 -; AVX512F-NEXT: vpshufb %xmm6, %xmm2, %xmm2 -; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0 +; AVX512F-NEXT: vmovdqa 112(%rdi), %xmm4 +; AVX512F-NEXT: vpshufb %xmm6, %xmm4, %xmm4 +; AVX512F-NEXT: vpor %xmm0, %xmm4, %xmm0 ; AVX512F-NEXT: vinserti32x4 $2, %xmm0, %zmm0, %zmm0 -; AVX512F-NEXT: vmovdqa 80(%rdi), %xmm2 -; AVX512F-NEXT: vpshufb %xmm1, %xmm2, %xmm1 -; AVX512F-NEXT: vmovdqa 64(%rdi), %xmm2 -; AVX512F-NEXT: vpshufb %xmm3, %xmm2, %xmm2 -; AVX512F-NEXT: vpor %xmm1, %xmm2, %xmm1 +; AVX512F-NEXT: vmovdqa 80(%rdi), %xmm4 +; AVX512F-NEXT: vpshufb %xmm1, %xmm4, %xmm1 +; AVX512F-NEXT: vmovdqa 64(%rdi), %xmm4 +; AVX512F-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1 ; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 ; AVX512F-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7] -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512BW-LABEL: f2: @@ -253,11 +251,9 @@ define <64 x i8> @f2(ptr %p0) { ; AVX512BW-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0 ; AVX512BW-NEXT: vmovdqa 32(%rdi), %ymm2 ; AVX512BW-NEXT: vinserti64x4 $1, 128(%rdi), %zmm2, %zmm2 -; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = -; AVX512BW-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3] ; AVX512BW-NEXT: movabsq $8998403163813888, %rax # imm = 0x1FF800001FF800 ; AVX512BW-NEXT: kmovq %rax, %k1 -; AVX512BW-NEXT: vpshufb %zmm4, %zmm2, %zmm0 {%k1} +; AVX512BW-NEXT: vpshufb {{.*#+}} zmm0 {%k1} = zmm2[u,u,u,u,u,u,u,u,u,u,u,3,5,9,11,15,17,21,23,27,29,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,35,37,41,43,47,49,53,55,59,61,u,u,u,u,u,u,u,u,u,u,u] ; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm2 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm2, %xmm2 ; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm4 @@ -501,23 +497,21 @@ define <64 x i8> @f4(ptr %p0) { ; AVX512F-NEXT: vmovdqa 128(%rdi), %ymm4 ; AVX512F-NEXT: vpshufb {{.*#+}} ymm4 = ymm4[u,u,u,u,u,u,u,u,u,u,u,2,4,8,10,14,16,20,22,26,28],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero ; AVX512F-NEXT: vinserti64x4 $1, %ymm4, %zmm2, %zmm2 -; AVX512F-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = [255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,0,0,0,0,0,0,0,0,0,0,255,255,255,255,255,255,255,255,255,255,255] -; AVX512F-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3] -; AVX512F-NEXT: vpternlogq $234, %zmm2, %zmm0, %zmm4 +; AVX512F-NEXT: vpternlogq $248, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm2 ; AVX512F-NEXT: vmovdqa 96(%rdi), %xmm0 ; AVX512F-NEXT: vpshufb %xmm5, %xmm0, %xmm0 -; AVX512F-NEXT: vmovdqa 112(%rdi), %xmm2 -; AVX512F-NEXT: vpshufb %xmm6, %xmm2, %xmm2 -; AVX512F-NEXT: vpor %xmm0, %xmm2, %xmm0 +; AVX512F-NEXT: vmovdqa 112(%rdi), %xmm4 +; AVX512F-NEXT: vpshufb %xmm6, %xmm4, %xmm4 +; AVX512F-NEXT: vpor %xmm0, %xmm4, %xmm0 ; AVX512F-NEXT: vinserti32x4 $2, %xmm0, %zmm0, %zmm0 -; AVX512F-NEXT: vmovdqa 80(%rdi), %xmm2 -; AVX512F-NEXT: vpshufb %xmm1, %xmm2, %xmm1 -; AVX512F-NEXT: vmovdqa 64(%rdi), %xmm2 -; AVX512F-NEXT: vpshufb %xmm3, %xmm2, %xmm2 -; AVX512F-NEXT: vpor %xmm1, %xmm2, %xmm1 +; AVX512F-NEXT: vmovdqa 80(%rdi), %xmm4 +; AVX512F-NEXT: vpshufb %xmm1, %xmm4, %xmm1 +; AVX512F-NEXT: vmovdqa 64(%rdi), %xmm4 +; AVX512F-NEXT: vpshufb %xmm3, %xmm4, %xmm3 +; AVX512F-NEXT: vpor %xmm1, %xmm3, %xmm1 ; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 ; AVX512F-NEXT: vshufi64x2 {{.*#+}} zmm0 = zmm1[0,1,2,3],zmm0[4,5,6,7] -; AVX512F-NEXT: vpternlogq $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm4, %zmm0 +; AVX512F-NEXT: vpternlogd $216, {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm2, %zmm0 ; AVX512F-NEXT: retq ; ; AVX512BW-LABEL: f4: @@ -540,11 +534,9 @@ define <64 x i8> @f4(ptr %p0) { ; AVX512BW-NEXT: vinserti64x4 $1, %ymm0, %zmm2, %zmm0 ; AVX512BW-NEXT: vmovdqa 32(%rdi), %ymm2 ; AVX512BW-NEXT: vinserti64x4 $1, 128(%rdi), %zmm2, %zmm2 -; AVX512BW-NEXT: vbroadcasti64x4 {{.*#+}} zmm4 = -; AVX512BW-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3] ; AVX512BW-NEXT: movabsq $8998403163813888, %rax # imm = 0x1FF800001FF800 ; AVX512BW-NEXT: kmovq %rax, %k1 -; AVX512BW-NEXT: vpshufb %zmm4, %zmm2, %zmm0 {%k1} +; AVX512BW-NEXT: vpshufb {{.*#+}} zmm0 {%k1} = zmm2[u,u,u,u,u,u,u,u,u,u,u,2,4,8,10,14,16,20,22,26,28,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,u,34,36,40,42,46,48,52,54,58,60,u,u,u,u,u,u,u,u,u,u,u] ; AVX512BW-NEXT: vmovdqa 96(%rdi), %xmm2 ; AVX512BW-NEXT: vpshufb %xmm5, %xmm2, %xmm2 ; AVX512BW-NEXT: vmovdqa 112(%rdi), %xmm4 diff --git a/llvm/test/CodeGen/X86/x86-interleaved-access.ll b/llvm/test/CodeGen/X86/x86-interleaved-access.ll index 216332943993..4f20a9d5db6a 100644 --- a/llvm/test/CodeGen/X86/x86-interleaved-access.ll +++ b/llvm/test/CodeGen/X86/x86-interleaved-access.ll @@ -1054,8 +1054,8 @@ define void @interleaved_store_vf32_i8_stride3(<32 x i8> %a, <32 x i8> %b, <32 x ; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm3 ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3] -; AVX512-NEXT: vbroadcasti64x4 {{.*#+}} zmm2 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] -; AVX512-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3] +; AVX512-NEXT: vbroadcasti32x4 {{.*#+}} zmm2 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] +; AVX512-NEXT: # zmm2 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3] ; AVX512-NEXT: vpshufb %ymm2, %ymm0, %ymm0 ; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm3, %zmm1 ; AVX512-NEXT: vpshufb %zmm2, %zmm1, %zmm1 @@ -1252,8 +1252,8 @@ define void @interleaved_store_vf64_i8_stride3(<64 x i8> %a, <64 x i8> %b, <64 x ; AVX512-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7] ; AVX512-NEXT: vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm2[2,3] ; AVX512-NEXT: vinserti64x4 $1, %ymm4, %zmm3, %zmm2 -; AVX512-NEXT: vbroadcasti64x4 {{.*#+}} zmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] -; AVX512-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3] +; AVX512-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5,0,11,6,1,12,7,2,13,8,3,14,9,4,15,10,5] +; AVX512-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3] ; AVX512-NEXT: vpshufb %zmm3, %zmm2, %zmm2 ; AVX512-NEXT: vinserti64x4 $1, %ymm6, %zmm5, %zmm4 ; AVX512-NEXT: vpshufb %zmm3, %zmm4, %zmm4 diff --git a/llvm/test/CodeGen/X86/zero-remat.ll b/llvm/test/CodeGen/X86/zero-remat.ll index 60bb2c420cda..000e0d14b711 100644 --- a/llvm/test/CodeGen/X86/zero-remat.ll +++ b/llvm/test/CodeGen/X86/zero-remat.ll @@ -19,11 +19,12 @@ define double @foo() nounwind { ; ; CHECK-32-LABEL: foo: ; CHECK-32: # %bb.0: -; CHECK-32-NEXT: pushl $0 -; CHECK-32-NEXT: pushl $0 +; CHECK-32-NEXT: subl $8, %esp +; CHECK-32-NEXT: fldz +; CHECK-32-NEXT: fstpl (%esp) ; CHECK-32-NEXT: calll bar@PLT -; CHECK-32-NEXT: addl $8, %esp ; CHECK-32-NEXT: fldz +; CHECK-32-NEXT: addl $8, %esp ; CHECK-32-NEXT: retl call void @bar(double 0.0) ret double 0.0 diff --git a/llvm/test/DebugInfo/assignment-tracking/X86/large-type.ll b/llvm/test/DebugInfo/assignment-tracking/X86/large-type.ll new file mode 100644 index 000000000000..cebbc162fcb3 --- /dev/null +++ b/llvm/test/DebugInfo/assignment-tracking/X86/large-type.ll @@ -0,0 +1,58 @@ +; RUN: llc %s -stop-after=finalize-isel -o - \ +; RUN: | FileCheck %s --implicit-check-not=DBG_ + +;; Based on optimized IR from C source: +;; int main () { +;; char a1[__INT_MAX__]; +;; a1[__INT_MAX__ - 1] = 5; +;; return a1[__INT_MAX__ - 1]; +;; } +;; +;; Check extremely large types don't cause a crash. +; CHECK: DBG_VALUE 5, $noreg, ![[#]], !DIExpression(DW_OP_LLVM_fragment, 4294967280, 8) +; CHECK: DBG_VALUE 6, $noreg, ![[#]], !DIExpression(DW_OP_LLVM_fragment, 0, 8) +; CHECK: DBG_VALUE 7, $noreg, ![[#]], !DIExpression(DW_OP_LLVM_fragment, 0, 8) + +define dso_local i32 @main() local_unnamed_addr !dbg !10 { +entry: +;; FIXME: SROA currently creates incorrect fragments if bit_offset > max(u32), +;; with and without assignment-tracking. + tail call void @llvm.dbg.value(metadata i8 5, metadata !15, metadata !DIExpression(DW_OP_LLVM_fragment, 4294967280, 8)), !dbg !20 +;; These two were inserted by hand. + tail call void @llvm.dbg.value(metadata i8 6, metadata !22, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !20 + tail call void @llvm.dbg.value(metadata i8 7, metadata !23, metadata !DIExpression(DW_OP_LLVM_fragment, 0, 8)), !dbg !20 + ret i32 5, !dbg !21 +} + +declare void @llvm.dbg.value(metadata, metadata, metadata) + +!llvm.dbg.cu = !{!0} +!llvm.module.flags = !{!2, !3, !4, !5, !6, !7, !8} +!llvm.ident = !{!9} + +!0 = distinct !DICompileUnit(language: DW_LANG_C11, file: !1, producer: "clang version 18.0.0", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug, splitDebugInlining: false, nameTableKind: None) +!1 = !DIFile(filename: "test.c", directory: "/") +!2 = !{i32 7, !"Dwarf Version", i32 5} +!3 = !{i32 2, !"Debug Info Version", i32 3} +!4 = !{i32 1, !"wchar_size", i32 4} +!5 = !{i32 8, !"PIC Level", i32 2} +!6 = !{i32 7, !"PIE Level", i32 2} +!7 = !{i32 7, !"uwtable", i32 2} +!8 = !{i32 7, !"debug-info-assignment-tracking", i1 true} +!9 = !{!"clang version 18.0.0"} +!10 = distinct !DISubprogram(name: "main", scope: !1, file: !1, line: 3, type: !11, scopeLine: 4, flags: DIFlagAllCallsDescribed, spFlags: DISPFlagDefinition | DISPFlagOptimized, unit: !0, retainedNodes: !14) +!11 = !DISubroutineType(types: !12) +!12 = !{!13} +!13 = !DIBasicType(name: "int", size: 32, encoding: DW_ATE_signed) +!14 = !{!15} +!15 = !DILocalVariable(name: "a1", scope: !10, file: !1, line: 5, type: !16) +!16 = !DICompositeType(tag: DW_TAG_array_type, baseType: !17, size: 17179869176, elements: !18) +!17 = !DIBasicType(name: "char", size: 8, encoding: DW_ATE_signed_char) +!18 = !{!19} +!19 = !DISubrange(count: 2147483647) +!20 = !DILocation(line: 0, scope: !10) +!21 = !DILocation(line: 7, column: 3, scope: !10) +!22 = !DILocalVariable(name: "a2", scope: !10, file: !1, line: 5, type: !16) +!23 = !DILocalVariable(name: "a3", scope: !10, file: !1, line: 5, type: !16) +!24 = !DICompositeType(tag: DW_TAG_array_type, baseType: !17, size: 4294967232, elements: !18) +!25 = !DICompositeType(tag: DW_TAG_array_type, baseType: !17, size: 4294967233, elements: !18) diff --git a/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1_elf.ll b/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1_elf.ll index 659dbe109ec6..0cdc5e7de844 100644 --- a/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1_elf.ll +++ b/llvm/test/Examples/OrcV2Examples/Inputs/argc_sub1_elf.ll @@ -1,5 +1,4 @@ ; ModuleID = 'argc_sub1.c' -target triple = "x86_64-unknown-unknown-elf" define i32 @sub1(i32) !dbg !8 { call void @llvm.dbg.value(metadata i32 %0, metadata !13, metadata !DIExpression()), !dbg !14 diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_metadata_code_model.ll b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_code_model.ll new file mode 100644 index 000000000000..c1e7694d4cd5 --- /dev/null +++ b/llvm/test/Instrumentation/AddressSanitizer/global_metadata_code_model.ll @@ -0,0 +1,10 @@ +; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -passes=asan -S | FileCheck %s --check-prefix=LARGE +; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -passes=asan -S | FileCheck %s --check-prefix=NORMAL +; RUN: opt < %s -mtriple=x86_64-pc-windows -passes=asan -S | FileCheck %s --check-prefix=NORMAL + +; check that asan globals metadata are emitted to a large section for x86-64 ELF + +; LARGE: @__asan_global_global = {{.*}}global {{.*}}, code_model "large" +; NORMAL-NOT: code_model "large" + +@global = global i32 0, align 4 diff --git a/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll b/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll index 699b8287d358..74f8fc9997d4 100644 --- a/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll +++ b/llvm/test/Instrumentation/AddressSanitizer/global_with_comdat.ll @@ -102,8 +102,8 @@ target triple = "x86_64-unknown-linux-gnu" ;; Don't place the instrumented globals in a comdat when the unique module ID is empty. ; NOMODULEID: @.str = internal constant { [4 x i8], [28 x i8] } { [4 x i8] c"str\00", [28 x i8] zeroinitializer }, align 32 ; NOMODULEID: @_ZL3buf = internal global { [4 x i8], [28 x i8] } zeroinitializer, align 32 -; NOMODULEID: @__asan_global_.str = private global {{.*}}, section "asan_globals", !associated !0 -; NOMODULEID: @__asan_global__ZL3buf = private global {{.*}}, section "asan_globals", !associated !1 +; NOMODULEID: @__asan_global_.str = private global {{.*}}, section "asan_globals"{{.*}}, !associated !0 +; NOMODULEID: @__asan_global__ZL3buf = private global {{.*}}, section "asan_globals"{{.*}}, !associated !1 ; NOMODULEID: @llvm.compiler.used = appending global [4 x ptr] [ptr @.str, ptr @_ZL3buf, ptr @__asan_global_.str, ptr @__asan_global__ZL3buf] ; NOMODULEID: define internal void @asan.module_ctor() #[[#]] comdat { diff --git a/llvm/test/Instrumentation/InstrProfiling/icall-comdat.ll b/llvm/test/Instrumentation/InstrProfiling/icall-comdat.ll index c90f37c82089..9fbff456ff50 100644 --- a/llvm/test/Instrumentation/InstrProfiling/icall-comdat.ll +++ b/llvm/test/Instrumentation/InstrProfiling/icall-comdat.ll @@ -1,5 +1,4 @@ ;; Check that static counters are allocated for value profiler - ; RUN: opt < %s -mtriple=x86_64-unknown-linux -passes=instrprof -vp-static-alloc=true -S | FileCheck %s --check-prefix=STATIC ; RUN: opt < %s -mtriple=powerpc-unknown-linux -passes=instrprof -vp-static-alloc=true -S | FileCheck %s --check-prefix=STATIC ; RUN: opt < %s -mtriple=sparc-unknown-linux -passes=instrprof -vp-static-alloc=true -S | FileCheck %s --check-prefix=STATIC @@ -16,6 +15,10 @@ ; RUN: opt %s -mtriple=powerpc64-ibm-aix -passes=instrprof -S | FileCheck %s --check-prefix=ALIGN ; RUN: opt %s -mtriple=x86_64-unknown-linux -passes=instrprof -S | FileCheck %s --check-prefix=ALIGN +;; Check that globals have the proper code model. +; RUN: opt %s -mtriple=x86_64-unknown-linux -passes=instrprof -S | FileCheck %s --check-prefixes=CODEMODEL,CODEMODEL-X8664 +; RUN: opt %s -mtriple=powerpc-unknown-linux -passes=instrprof -S | FileCheck %s --check-prefixes=CODEMODEL,CODEMODEL-PPC + @__profn_foo = private constant [3 x i8] c"foo" @__profn_bar = private constant [3 x i8] c"bar" @@ -46,8 +49,8 @@ declare void @llvm.instrprof.value.profile(ptr, i64, i64, i32, i32) #0 attributes #0 = { nounwind } -; STATIC: @__profvp_foo = private global [1 x i64] zeroinitializer, section "{{[^"]+}}", comdat($__profc_foo) -; STATIC: @__profvp_bar = private global [1 x i64] zeroinitializer, section "{{[^"]+}}", comdat($__profc_bar) +; STATIC: @__profvp_foo = private global [1 x i64] zeroinitializer, section "{{[^"]+}}",{{.*}} comdat($__profc_foo) +; STATIC: @__profvp_bar = private global [1 x i64] zeroinitializer, section "{{[^"]+}}",{{.*}} comdat($__profc_bar) ; STATIC: @__llvm_prf_vnodes ; DYN-NOT: @__profvp_foo @@ -73,5 +76,26 @@ attributes #0 = { nounwind } ; ALIGN: @__profc_bar = private global {{.*}} section "__llvm_prf_cnts",{{.*}} align 8 ; ALIGN: @__profvp_bar = private global {{.*}} section "__llvm_prf_vals",{{.*}} align 8 ; ALIGN: @__profd_bar = private global {{.*}} section "__llvm_prf_data",{{.*}} align 8 -; ALIGN: @__llvm_prf_vnodes = private global {{.*}} section "__llvm_prf_vnds", align 8 -; ALIGN: @__llvm_prf_nm = private constant {{.*}} section "__llvm_prf_names", align 1 +; ALIGN: @__llvm_prf_vnodes = private global {{.*}} section "__llvm_prf_vnds",{{.*}} align 8 +; ALIGN: @__llvm_prf_nm = private constant {{.*}} section "__llvm_prf_names",{{.*}} align 1 + +; CODEMODEL: @__profc_foo = +; CODEMODEL-NOT: code_model "large" +; CODEMODEL: @__profvp_foo = +; CODEMODEL-X8664-SAME: code_model "large" +; CODEMODEL-PPC-NOT: code_model +; CODEMODEL: @__profd_foo = +; CODEMODEL-NOT: code_model "large" +; CODEMODEL: @__profc_bar = +; CODEMODEL-NOT: code_model "large" +; CODEMODEL: @__profvp_bar = +; CODEMODEL-X8664-SAME: code_model "large" +; CODEMODEL-PPC-NOT: code_model +; CODEMODEL: @__profd_bar = +; CODEMODEL-NOT: code_model "large" +; CODEMODEL: @__llvm_prf_vnodes = +; CODEMODEL-X8664-SAME: code_model "large" +; CODEMODEL-PPC-NOT: code_model +; CODEMODEL: @__llvm_prf_nm = +; CODEMODEL-X8664-SAME: code_model "large" +; CODEMODEL-PPC-NOT: code_model diff --git a/llvm/test/Instrumentation/InstrProfiling/platform.ll b/llvm/test/Instrumentation/InstrProfiling/platform.ll index 3c3712240d1e..81912f3c0b6a 100644 --- a/llvm/test/Instrumentation/InstrProfiling/platform.ll +++ b/llvm/test/Instrumentation/InstrProfiling/platform.ll @@ -25,7 +25,7 @@ ; WINDOWS: @__profd_foo = private global {{.*}}, section ".lprfd$M", align 8 ; AIX: @__profd_foo = private {{.*}}, section "__llvm_prf_data", align 8 -; ELF: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}__llvm_prf_names", align 1 +; ELF: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}__llvm_prf_names"{{.*}}, align 1 ; WINDOWS: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}lprfn$M", align 1 ; AIX: @__llvm_prf_nm = private constant [{{.*}} x i8] c"{{.*}}", section "{{.*}}__llvm_prf_names", align 1 diff --git a/llvm/test/MC/AArch64/elf-reloc-ptrauth.s b/llvm/test/MC/AArch64/elf-reloc-ptrauth.s index 1ce008117ac3..3bd8f5c19932 100644 --- a/llvm/test/MC/AArch64/elf-reloc-ptrauth.s +++ b/llvm/test/MC/AArch64/elf-reloc-ptrauth.s @@ -5,14 +5,14 @@ // RELOC: Relocation section '.rela.test' at offset 0x230 contains 8 entries: // RELOC-NEXT: Offset Info Type Symbol's Value Symbol's Name + Addend -// RELOC-NEXT: 0000000000000000 000000010000e100 R_AARCH64_AUTH_ABS64 0000000000000000 .helper + 0 -// RELOC-NEXT: 0000000000000010 000000080000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g1 + 0 -// RELOC-NEXT: 0000000000000020 000000090000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g2 + 0 -// RELOC-NEXT: 0000000000000030 0000000a0000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g3 + 0 -// RELOC-NEXT: 0000000000000040 0000000b0000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g4 + 7 -// RELOC-NEXT: 0000000000000050 0000000c0000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g5 - 3 -// RELOC-NEXT: 0000000000000060 000000020000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g 6 + 0 -// RELOC-NEXT: 0000000000000070 0000000d0000e100 R_AARCH64_AUTH_ABS64 0000000000000000 _g 7 + 7 +// RELOC-NEXT: 0000000000000000 0000000100000244 R_AARCH64_AUTH_ABS64 0000000000000000 .helper + 0 +// RELOC-NEXT: 0000000000000010 0000000800000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g1 + 0 +// RELOC-NEXT: 0000000000000020 0000000900000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g2 + 0 +// RELOC-NEXT: 0000000000000030 0000000a00000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g3 + 0 +// RELOC-NEXT: 0000000000000040 0000000b00000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g4 + 7 +// RELOC-NEXT: 0000000000000050 0000000c00000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g5 - 3 +// RELOC-NEXT: 0000000000000060 0000000200000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g 6 + 0 +// RELOC-NEXT: 0000000000000070 0000000d00000244 R_AARCH64_AUTH_ABS64 0000000000000000 _g 7 + 7 // RELOC: Hex dump of section '.test': // VVVVVVVV addend, not needed for rela diff --git a/llvm/test/MC/AMDGPU/exp.s b/llvm/test/MC/AMDGPU/exp.s index a2fa09b71496..d04e15202a58 100644 --- a/llvm/test/MC/AMDGPU/exp.s +++ b/llvm/test/MC/AMDGPU/exp.s @@ -3,103 +3,124 @@ // RUN: llvm-mc -triple=amdgcn -mcpu=gfx900 -show-encoding %s | FileCheck -check-prefix=GFX89 %s // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1010 -show-encoding %s | FileCheck -check-prefix=GFX10 %s // RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck -check-prefix=GFX10 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX12 %s exp mrt0 off, off, off, off // SI: exp mrt0 off, off, off, off ; encoding: [0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x00] // GFX89: exp mrt0 off, off, off, off ; encoding: [0x00,0x00,0x00,0xc4,0x00,0x00,0x00,0x00] // GFX10: exp mrt0 off, off, off, off ; encoding: [0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x00] +// GFX12: export mrt0 off, off, off, off ; encoding: [0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x00] exp mrt0 off, off, off, off done // SI: exp mrt0 off, off, off, off done ; encoding: [0x00,0x08,0x00,0xf8,0x00,0x00,0x00,0x00] // GFX89: exp mrt0 off, off, off, off done ; encoding: [0x00,0x08,0x00,0xc4,0x00,0x00,0x00,0x00] // GFX10: exp mrt0 off, off, off, off done ; encoding: [0x00,0x08,0x00,0xf8,0x00,0x00,0x00,0x00] +// GFX12: export mrt0 off, off, off, off done ; encoding: [0x00,0x08,0x00,0xf8,0x00,0x00,0x00,0x00] exp mrt0 v4, off, off, off done // SI: exp mrt0 v4, off, off, off done ; encoding: [0x01,0x08,0x00,0xf8,0x04,0x00,0x00,0x00] // GFX89: exp mrt0 v4, off, off, off done ; encoding: [0x01,0x08,0x00,0xc4,0x04,0x00,0x00,0x00] // GFX10: exp mrt0 v4, off, off, off done ; encoding: [0x01,0x08,0x00,0xf8,0x04,0x00,0x00,0x00] +// GFX12: export mrt0 v4, off, off, off done ; encoding: [0x01,0x08,0x00,0xf8,0x04,0x00,0x00,0x00] exp mrt0 off, v3, off, off done // SI: exp mrt0 off, v3, off, off done ; encoding: [0x02,0x08,0x00,0xf8,0x00,0x03,0x00,0x00] // GFX89: exp mrt0 off, v3, off, off done ; encoding: [0x02,0x08,0x00,0xc4,0x00,0x03,0x00,0x00] // GFX10: exp mrt0 off, v3, off, off done ; encoding: [0x02,0x08,0x00,0xf8,0x00,0x03,0x00,0x00] +// GFX12: export mrt0 off, v3, off, off done ; encoding: [0x02,0x08,0x00,0xf8,0x00,0x03,0x00,0x00] exp mrt0 off, off, v2, off done // SI: exp mrt0 off, off, v2, off done ; encoding: [0x04,0x08,0x00,0xf8,0x00,0x00,0x02,0x00] // GFX89: exp mrt0 off, off, v2, off done ; encoding: [0x04,0x08,0x00,0xc4,0x00,0x00,0x02,0x00] // GFX10: exp mrt0 off, off, v2, off done ; encoding: [0x04,0x08,0x00,0xf8,0x00,0x00,0x02,0x00] +// GFX12: export mrt0 off, off, v2, off done ; encoding: [0x04,0x08,0x00,0xf8,0x00,0x00,0x02,0x00] exp mrt0 off, off, off, v1 done // SI: exp mrt0 off, off, off, v1 done ; encoding: [0x08,0x08,0x00,0xf8,0x00,0x00,0x00,0x01] // GFX89: exp mrt0 off, off, off, v1 done ; encoding: [0x08,0x08,0x00,0xc4,0x00,0x00,0x00,0x01] // GFX10: exp mrt0 off, off, off, v1 done ; encoding: [0x08,0x08,0x00,0xf8,0x00,0x00,0x00,0x01] +// GFX12: export mrt0 off, off, off, v1 done ; encoding: [0x08,0x08,0x00,0xf8,0x00,0x00,0x00,0x01] exp mrt0 v4, v3, off, off done // SI: exp mrt0 v4, v3, off, off done ; encoding: [0x03,0x08,0x00,0xf8,0x04,0x03,0x00,0x00] // GFX89: exp mrt0 v4, v3, off, off done ; encoding: [0x03,0x08,0x00,0xc4,0x04,0x03,0x00,0x00] // GFX10: exp mrt0 v4, v3, off, off done ; encoding: [0x03,0x08,0x00,0xf8,0x04,0x03,0x00,0x00] +// GFX12: export mrt0 v4, v3, off, off done ; encoding: [0x03,0x08,0x00,0xf8,0x04,0x03,0x00,0x00] exp mrt0 v4, off, v2, off done // SI: exp mrt0 v4, off, v2, off done ; encoding: [0x05,0x08,0x00,0xf8,0x04,0x00,0x02,0x00] // GFX89: exp mrt0 v4, off, v2, off done ; encoding: [0x05,0x08,0x00,0xc4,0x04,0x00,0x02,0x00] // GFX10: exp mrt0 v4, off, v2, off done ; encoding: [0x05,0x08,0x00,0xf8,0x04,0x00,0x02,0x00] +// GFX12: export mrt0 v4, off, v2, off done ; encoding: [0x05,0x08,0x00,0xf8,0x04,0x00,0x02,0x00] exp mrt0 v4, off, off, v1 // SI: exp mrt0 v4, off, off, v1 ; encoding: [0x09,0x00,0x00,0xf8,0x04,0x00,0x00,0x01] // GFX89: exp mrt0 v4, off, off, v1 ; encoding: [0x09,0x00,0x00,0xc4,0x04,0x00,0x00,0x01] // GFX10: exp mrt0 v4, off, off, v1 ; encoding: [0x09,0x00,0x00,0xf8,0x04,0x00,0x00,0x01] +// GFX12: export mrt0 v4, off, off, v1 ; encoding: [0x09,0x00,0x00,0xf8,0x04,0x00,0x00,0x01] exp mrt0 v4, off, off, v1 done // SI: exp mrt0 v4, off, off, v1 done ; encoding: [0x09,0x08,0x00,0xf8,0x04,0x00,0x00,0x01] // GFX89: exp mrt0 v4, off, off, v1 done ; encoding: [0x09,0x08,0x00,0xc4,0x04,0x00,0x00,0x01] // GFX10: exp mrt0 v4, off, off, v1 done ; encoding: [0x09,0x08,0x00,0xf8,0x04,0x00,0x00,0x01] +// GFX12: export mrt0 v4, off, off, v1 done ; encoding: [0x09,0x08,0x00,0xf8,0x04,0x00,0x00,0x01] exp mrt0 v4, v3, v2, v1 // SI: exp mrt0 v4, v3, v2, v1 ; encoding: [0x0f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp mrt0 v4, v3, v2, v1 ; encoding: [0x0f,0x00,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp mrt0 v4, v3, v2, v1 ; encoding: [0x0f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export mrt0 v4, v3, v2, v1 ; encoding: [0x0f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] exp mrt0 v4, v3, v2, v1 done // SI: exp mrt0 v4, v3, v2, v1 done ; encoding: [0x0f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp mrt0 v4, v3, v2, v1 done ; encoding: [0x0f,0x08,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp mrt0 v4, v3, v2, v1 done ; encoding: [0x0f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export mrt0 v4, v3, v2, v1 done ; encoding: [0x0f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] exp mrt7 v1, v1, v1, v1 // SI: exp mrt7 v1, v1, v1, v1 ; encoding: [0x7f,0x00,0x00,0xf8,0x01,0x01,0x01,0x01] // GFX89: exp mrt7 v1, v1, v1, v1 ; encoding: [0x7f,0x00,0x00,0xc4,0x01,0x01,0x01,0x01] // GFX10: exp mrt7 v1, v1, v1, v1 ; encoding: [0x7f,0x00,0x00,0xf8,0x01,0x01,0x01,0x01] +// GFX12: export mrt7 v1, v1, v1, v1 ; encoding: [0x7f,0x00,0x00,0xf8,0x01,0x01,0x01,0x01] exp mrt7 v1, v1, v1, v1 done // SI: exp mrt7 v1, v1, v1, v1 done ; encoding: [0x7f,0x08,0x00,0xf8,0x01,0x01,0x01,0x01] // GFX89: exp mrt7 v1, v1, v1, v1 done ; encoding: [0x7f,0x08,0x00,0xc4,0x01,0x01,0x01,0x01] // GFX10: exp mrt7 v1, v1, v1, v1 done ; encoding: [0x7f,0x08,0x00,0xf8,0x01,0x01,0x01,0x01] +// GFX12: export mrt7 v1, v1, v1, v1 done ; encoding: [0x7f,0x08,0x00,0xf8,0x01,0x01,0x01,0x01] exp mrtz v4, v3, v2, v1 // SI: exp mrtz v4, v3, v2, v1 ; encoding: [0x8f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp mrtz v4, v3, v2, v1 ; encoding: [0x8f,0x00,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp mrtz v4, v3, v2, v1 ; encoding: [0x8f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export mrtz v4, v3, v2, v1 ; encoding: [0x8f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] exp mrtz v4, v3, v2, v1 done // SI: exp mrtz v4, v3, v2, v1 done ; encoding: [0x8f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp mrtz v4, v3, v2, v1 done ; encoding: [0x8f,0x08,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp mrtz v4, v3, v2, v1 done ; encoding: [0x8f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export mrtz v4, v3, v2, v1 done ; encoding: [0x8f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] exp pos0 v4, v3, v2, v1 // SI: exp pos0 v4, v3, v2, v1 ; encoding: [0xcf,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp pos0 v4, v3, v2, v1 ; encoding: [0xcf,0x00,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp pos0 v4, v3, v2, v1 ; encoding: [0xcf,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export pos0 v4, v3, v2, v1 ; encoding: [0xcf,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] exp pos0 v4, v3, v2, v1 done // SI: exp pos0 v4, v3, v2, v1 done ; encoding: [0xcf,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp pos0 v4, v3, v2, v1 done ; encoding: [0xcf,0x08,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp pos0 v4, v3, v2, v1 done ; encoding: [0xcf,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export pos0 v4, v3, v2, v1 done ; encoding: [0xcf,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] exp pos3 v4, v3, v2, v1 // SI: exp pos3 v4, v3, v2, v1 ; encoding: [0xff,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp pos3 v4, v3, v2, v1 ; encoding: [0xff,0x00,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp pos3 v4, v3, v2, v1 ; encoding: [0xff,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export pos3 v4, v3, v2, v1 ; encoding: [0xff,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] exp pos3 v4, v3, v2, v1 done // SI: exp pos3 v4, v3, v2, v1 done ; encoding: [0xff,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] // GFX89: exp pos3 v4, v3, v2, v1 done ; encoding: [0xff,0x08,0x00,0xc4,0x04,0x03,0x02,0x01] // GFX10: exp pos3 v4, v3, v2, v1 done ; encoding: [0xff,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] +// GFX12: export pos3 v4, v3, v2, v1 done ; encoding: [0xff,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s index e2e53776783f..fdfbf65c0e3c 100644 --- a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s +++ b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp.s @@ -1,277 +1,278 @@ -// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck -check-prefix=GFX11 %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1100 -show-encoding %s | FileCheck -check-prefix=GCN %s +// RUN: llvm-mc -triple=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GCN %s v_interp_p10_f32 v0, v1, v2, v3 -// GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f32 v1, v10, v20, v30 -// GFX11: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04] +// GCN: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04] v_interp_p10_f32 v2, v11, v21, v31 -// GFX11: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04] +// GCN: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04] v_interp_p10_f32 v3, v12, v22, v32 -// GFX11: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04] +// GCN: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04] v_interp_p10_f32 v0, v1, v2, v3 clamp -// GFX11: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f32 v0, -v1, v2, v3 -// GFX11: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24] +// GCN: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24] v_interp_p10_f32 v0, v1, -v2, v3 -// GFX11: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44] +// GCN: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44] v_interp_p10_f32 v0, v1, v2, -v3 -// GFX11: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84] +// GCN: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84] v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 -// GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 -// GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 -// GFX11: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 -// GFX11: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f32 v0, v1, v2, v3 -// GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f32 v1, v10, v20, v30 -// GFX11: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04] +// GCN: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0 ; encoding: [0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04] v_interp_p2_f32 v2, v11, v21, v31 -// GFX11: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04] +// GCN: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0 ; encoding: [0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04] v_interp_p2_f32 v3, v12, v22, v32 -// GFX11: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04] +// GCN: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0 ; encoding: [0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04] v_interp_p2_f32 v0, v1, v2, v3 clamp -// GFX11: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f32 v0, -v1, v2, v3 -// GFX11: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24] +// GCN: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24] v_interp_p2_f32 v0, v1, -v2, v3 -// GFX11: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44] +// GCN: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44] v_interp_p2_f32 v0, v1, v2, -v3 -// GFX11: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84] +// GCN: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84] v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 -// GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 -// GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 -// GFX11: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 -// GFX11: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7 ; encoding: [0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, -v1, v2, v3 -// GFX11: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24] +// GCN: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24] v_interp_p10_f16_f32 v0, v1, -v2, v3 -// GFX11: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44] +// GCN: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44] v_interp_p10_f16_f32 v0, v1, v2, -v3 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84] v_interp_p10_f16_f32 v0, v1, v2, v3 clamp -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4] +// GCN: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4] v_interp_p2_f16_f32 v0, v1, v2, v3 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, -v1, v2, v3 -// GFX11: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24] +// GCN: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24] v_interp_p2_f16_f32 v0, v1, -v2, v3 -// GFX11: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44] +// GCN: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44] v_interp_p2_f16_f32 v0, v1, v2, -v3 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84] v_interp_p2_f16_f32 v0, v1, v2, v3 clamp -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4] +// GCN: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 -// GFX11: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24] +// GCN: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24] v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44] v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04] v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4] +// GCN: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 -// GFX11: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24] +// GCN: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24] v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44] v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0 ; encoding: [0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1 ; encoding: [0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7 ; encoding: [0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,0] -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0 ; encoding: [0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0 ; encoding: [0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0 ; encoding: [0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0 ; encoding: [0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0 ; encoding: [0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0 ; encoding: [0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04] +// GCN: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04] v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 -// GFX11: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4] +// GCN: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5 ; encoding: [0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4] diff --git a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err.s b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err.s index 1d6769dbd4d6..66b04c0d9086 100644 --- a/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err.s +++ b/llvm/test/MC/AMDGPU/gfx11_asm_vinterp_err.s @@ -1,4 +1,5 @@ -// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 %s 2>&1 | FileCheck %s -check-prefix=GFX11-ERR --implicit-check-not=error: --strict-whitespace +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 %s 2>&1 | FileCheck %s -check-prefix=GCN-ERR --implicit-check-not=error: --strict-whitespace +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck %s -check-prefix=GCN-ERR --implicit-check-not=error: --strict-whitespace //===----------------------------------------------------------------------===// // VINTERP src operands must be VGPRs. @@ -6,37 +7,37 @@ //===----------------------------------------------------------------------===// v_interp_p10_f32 v0, s1, v2, v3 -// GFX11-ERR: :[[@LINE-1]]:22: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:22: error: invalid operand for instruction v_interp_p10_f32 v0, v1, s2, v3 -// GFX11-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction v_interp_p10_f32 v0, v1, v2, s3 -// GFX11-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction v_interp_p2_f32 v0, 1, v2, v3 -// GFX11-ERR: :[[@LINE-1]]:21: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:21: error: invalid operand for instruction v_interp_p2_f32 v0, v1, 2, v3 -// GFX11-ERR: :[[@LINE-1]]:25: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:25: error: invalid operand for instruction v_interp_p2_f32 v0, v1, v2, 3 -// GFX11-ERR: :[[@LINE-1]]:29: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:29: error: invalid operand for instruction v_interp_p10_f16_f32 v0, s1, v2, v3 -// GFX11-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:26: error: invalid operand for instruction v_interp_p10_f16_f32 v0, v1, s2, v3 -// GFX11-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:30: error: invalid operand for instruction v_interp_p10_f16_f32 v0, v1, v2, s3 -// GFX11-ERR: :[[@LINE-1]]:34: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:34: error: invalid operand for instruction v_interp_p2_f16_f32 v0, 1, v2, v3 -// GFX11-ERR: :[[@LINE-1]]:25: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:25: error: invalid operand for instruction v_interp_p2_f16_f32 v0, v1, 2, v3 -// GFX11-ERR: :[[@LINE-1]]:29: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:29: error: invalid operand for instruction v_interp_p2_f16_f32 v0, v1, v2, 3 -// GFX11-ERR: :[[@LINE-1]]:33: error: invalid operand for instruction +// GCN-ERR: :[[@LINE-1]]:33: error: invalid operand for instruction diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_exp.s b/llvm/test/MC/AMDGPU/gfx12_asm_exp.s new file mode 100644 index 000000000000..2390813954a2 --- /dev/null +++ b/llvm/test/MC/AMDGPU/gfx12_asm_exp.s @@ -0,0 +1,61 @@ +// RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -show-encoding %s | FileCheck -check-prefix=GFX12 %s + +export mrt0 off, off, off, off +// GFX12: export mrt0 off, off, off, off ; encoding: [0x00,0x00,0x00,0xf8,0x00,0x00,0x00,0x00] + +export mrt0 off, off, off, off done +// GFX12: export mrt0 off, off, off, off done ; encoding: [0x00,0x08,0x00,0xf8,0x00,0x00,0x00,0x00] + +export mrt0 v4, off, off, off done +// GFX12: export mrt0 v4, off, off, off done ; encoding: [0x01,0x08,0x00,0xf8,0x04,0x00,0x00,0x00] + +export mrt0 off, v3, off, off done +// GFX12: export mrt0 off, v3, off, off done ; encoding: [0x02,0x08,0x00,0xf8,0x00,0x03,0x00,0x00] + +export mrt0 off, off, v2, off done +// GFX12: export mrt0 off, off, v2, off done ; encoding: [0x04,0x08,0x00,0xf8,0x00,0x00,0x02,0x00] + +export mrt0 off, off, off, v1 done +// GFX12: export mrt0 off, off, off, v1 done ; encoding: [0x08,0x08,0x00,0xf8,0x00,0x00,0x00,0x01] + +export mrt0 v4, v3, off, off done +// GFX12: export mrt0 v4, v3, off, off done ; encoding: [0x03,0x08,0x00,0xf8,0x04,0x03,0x00,0x00] + +export mrt0 v4, off, v2, off done +// GFX12: export mrt0 v4, off, v2, off done ; encoding: [0x05,0x08,0x00,0xf8,0x04,0x00,0x02,0x00] + +export mrt0 v4, off, off, v1 +// GFX12: export mrt0 v4, off, off, v1 ; encoding: [0x09,0x00,0x00,0xf8,0x04,0x00,0x00,0x01] + +export mrt0 v4, off, off, v1 done +// GFX12: export mrt0 v4, off, off, v1 done ; encoding: [0x09,0x08,0x00,0xf8,0x04,0x00,0x00,0x01] + +export mrt0 v4, v3, v2, v1 +// GFX12: export mrt0 v4, v3, v2, v1 ; encoding: [0x0f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] + +export mrt0 v4, v3, v2, v1 done +// GFX12: export mrt0 v4, v3, v2, v1 done ; encoding: [0x0f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] + +export mrt7 v1, v1, v1, v1 +// GFX12: export mrt7 v1, v1, v1, v1 ; encoding: [0x7f,0x00,0x00,0xf8,0x01,0x01,0x01,0x01] + +export mrt7 v1, v1, v1, v1 done +// GFX12: export mrt7 v1, v1, v1, v1 done ; encoding: [0x7f,0x08,0x00,0xf8,0x01,0x01,0x01,0x01] + +export mrtz v4, v3, v2, v1 +// GFX12: export mrtz v4, v3, v2, v1 ; encoding: [0x8f,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] + +export mrtz v4, v3, v2, v1 done +// GFX12: export mrtz v4, v3, v2, v1 done ; encoding: [0x8f,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] + +export pos0 v4, v3, v2, v1 +// GFX12: export pos0 v4, v3, v2, v1 ; encoding: [0xcf,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] + +export pos0 v4, v3, v2, v1 done +// GFX12: export pos0 v4, v3, v2, v1 done ; encoding: [0xcf,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] + +export pos3 v4, v3, v2, v1 +// GFX12: export pos3 v4, v3, v2, v1 ; encoding: [0xff,0x00,0x00,0xf8,0x04,0x03,0x02,0x01] + +export pos3 v4, v3, v2, v1 done +// GFX12: export pos3 v4, v3, v2, v1 done ; encoding: [0xff,0x08,0x00,0xf8,0x04,0x03,0x02,0x01] diff --git a/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s b/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s index 494b8399a26f..db166e8ffc10 100644 --- a/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s +++ b/llvm/test/MC/AMDGPU/gfx12_asm_sop1.s @@ -3648,6 +3648,9 @@ s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_SAVE_WAVE) s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TBA) // GFX12: encoding: [0x85,0x4c,0x80,0xbe] +s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_SE_AID_ID) +// GFX12: encoding: [0x86,0x4c,0x80,0xbe] + s_ctz_i32_b32 s5, s1 // GFX12: encoding: [0x01,0x08,0x85,0xbe] diff --git a/llvm/test/MC/AMDGPU/sopp-err.s b/llvm/test/MC/AMDGPU/sopp-err.s index c7f28faa5576..bd044cb74340 100644 --- a/llvm/test/MC/AMDGPU/sopp-err.s +++ b/llvm/test/MC/AMDGPU/sopp-err.s @@ -2,7 +2,8 @@ // RUN: not llvm-mc -triple=amdgcn -mcpu=tahiti %s 2>&1 | FileCheck --check-prefixes=GCN,PREGFX11,SICI,SICIVI --implicit-check-not=error: %s // RUN: not llvm-mc -triple=amdgcn -mcpu=fiji %s 2>&1 | FileCheck --check-prefixes=GCN,PREGFX11,VI,SICIVI --implicit-check-not=error: %s // RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1010 %s 2>&1 | FileCheck --check-prefixes=GCN,PREGFX11,GFX10 --implicit-check-not=error: %s -// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 %s 2>&1 | FileCheck --check-prefixes=GCN,GFX11 --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1100 %s 2>&1 | FileCheck --check-prefixes=GCN,GFX11PLUS,GFX11 --implicit-check-not=error: %s +// RUN: not llvm-mc -triple=amdgcn -mcpu=gfx1200 %s 2>&1 | FileCheck --check-prefixes=GCN,GFX11PLUS,GFX12 --implicit-check-not=error: %s //===----------------------------------------------------------------------===// // sendmsg @@ -22,30 +23,30 @@ s_sendmsg sendmsg(MSG_INTERRUPT, 0, 0) s_sendmsg sendmsg(MSG_GS) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: missing message operation -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS, GS_OP_NOP) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS, SYSMSG_OP_ECC_ERR_INTERRUPT) // GCN: :[[@LINE-1]]:{{[0-9]+}}: error: expected an operation name or an absolute expression s_sendmsg sendmsg(MSG_GS, 0) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS, -1) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS, 4) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS, 8) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(15, -1) // GCN: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operation id @@ -67,7 +68,7 @@ s_sendmsg sendmsg(MSG_GS, 1 -) s_sendmsg sendmsg(MSG_GS, GS_OP_CUT, 4) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: invalid message stream id -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS, GS_OP_CUT, 1 -) // GCN: :[[@LINE-1]]:{{[0-9]+}}: error: unknown token in expression @@ -86,11 +87,11 @@ s_sendmsg sendmsg(2, 2, 0, 0) s_sendmsg sendmsg(MSG_GS_DONE, GS_OP_NOP, 0) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: message operation does not support streams -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS_DONE, 0, 0) // PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: message operation does not support streams -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_HS_TESSFACTOR) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU @@ -104,25 +105,27 @@ s_sendmsg sendmsg(MSG_DEALLOC_VGPRS) s_sendmsg sendmsg(MSG_SAVEWAVE) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_STALL_WAVE_GEN) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX12: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_HALT_WAVES) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX12: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_ORDERED_PS_DONE) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU -// GFX11: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_EARLY_PRIM_DEALLOC) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU -// GFX11: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GS_ALLOC_REQ) // VI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU @@ -132,17 +135,17 @@ s_sendmsg sendmsg(MSG_GS_ALLOC_REQ, 0) // VI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // SICI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // GFX10: :[[@LINE-3]]:{{[0-9]+}}: error: message does not support operations -// GFX11: :[[@LINE-4]]:{{[0-9]+}}: error: message does not support operations +// GFX11PLUS: :[[@LINE-4]]:{{[0-9]+}}: error: message does not support operations s_sendmsg sendmsg(MSG_GET_DOORBELL) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU -// GFX11: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_GET_DDID) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU -// GFX11: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11PLUS: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU s_sendmsg sendmsg(MSG_RTN_GET_DOORBELL) // SICI: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU @@ -174,6 +177,10 @@ s_sendmsg sendmsg(MSG_RTN_GET_TBA) // VI: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU // GFX10: :[[@LINE-3]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +s_sendmsg sendmsg(MSG_RTN_GET_SE_AID_ID) +// PREGFX11: :[[@LINE-1]]:{{[0-9]+}}: error: specified message id is not supported on this GPU +// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: specified message id is not supported on this GPU + s_sendmsg sendmsg(-1) // GCN: :[[@LINE-1]]:{{[0-9]+}}: error: invalid message id @@ -252,12 +259,12 @@ s_waitcnt vmcnt(0 s_waitcnt_depctr 65536 // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr -32769 // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid operand for instruction -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid operand for instruction // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_hold_cnt(0) @@ -266,162 +273,162 @@ s_waitcnt_depctr depctr_hold_cnt(0) s_waitcnt_depctr depctr_sa_sdst(-1) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_sa_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_sa_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_sa_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vdst(-1) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_vdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(-1) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_ssrc(-1) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_ssrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_ssrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_ssrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vcc(-1) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_vcc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vcc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vcc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(-1) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_vm_vsrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_vm_vsrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_vm_vsrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_sa_sdst(2) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_sa_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_sa_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_sa_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vdst(16) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_vdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(8) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_ssrc(2) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_ssrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_ssrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_ssrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vcc(2) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_va_vcc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vcc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_va_vcc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(8) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid value for depctr_vm_vsrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_vm_vsrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid value for depctr_vm_vsrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_(8) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: invalid counter name depctr_vm_ -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: invalid counter name depctr_vm_ +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: invalid counter name depctr_vm_ // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_sa_sdst(0) depctr_sa_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_sa_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_sa_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_sa_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vdst(0) depctr_va_vdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(0) depctr_va_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_va_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_ssrc(0) depctr_va_ssrc(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_va_ssrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_ssrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_ssrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vcc(0) depctr_va_vcc(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vcc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vcc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vcc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(0) depctr_vm_vsrc(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_vm_vsrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_vm_vsrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_vm_vsrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_sa_sdst(0) depctr_va_sdst(0) depctr_sa_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_sa_sdst -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_sa_sdst +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_sa_sdst // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_ssrc(0) depctr_va_sdst(0) depctr_va_ssrc(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_va_ssrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_ssrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_ssrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_vcc(0) depctr_va_vcc(0) depctr_va_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vcc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vcc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_va_vcc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(0) depctr_vm_vsrc(0) depctr_va_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: duplicate counter name depctr_vm_vsrc -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_vm_vsrc +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: duplicate counter name depctr_vm_vsrc // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(0) depctr_vm_vsrc 0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a left parenthesis -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a left parenthesis +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a left parenthesis // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(0) 0depctr_vm_vsrc(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a counter name -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(0) depctr_vm_vsrc(x) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected absolute expression -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected absolute expression +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected absolute expression // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_va_sdst(0) depctr_vm_vsrc(0; & depctr_va_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a closing parenthesis -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a closing parenthesis +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a closing parenthesis // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc 0) depctr_vm_vsrc(0) depctr_va_sdst(0) // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected absolute expression -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected absolute expression +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected absolute expression // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(0) , // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a counter name -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(0) , & // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a counter name -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(0) & // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a counter name -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU s_waitcnt_depctr depctr_vm_vsrc(0) & & // GFX10: :[[@LINE-1]]:{{[0-9]+}}: error: expected a counter name -// GFX11: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name +// GFX11PLUS: :[[@LINE-2]]:{{[0-9]+}}: error: expected a counter name // SICIVI: :[[@LINE-3]]:{{[0-9]+}}: error: instruction not supported on this GPU //===----------------------------------------------------------------------===// diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt new file mode 100644 index 000000000000..3f173123983b --- /dev/null +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_exp.txt @@ -0,0 +1,13 @@ +# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble -show-encoding < %s | FileCheck %s -check-prefix=GFX12 + +# GFX12: export dual_src_blend0 v4, v3, v2, v1 ; encoding: [0x5f,0x01,0x00,0xf8,0x04,0x03,0x02,0x01] +0x5f,0x01,0x00,0xf8,0x04,0x03,0x02,0x01 + +# GFX12: export dual_src_blend1 v2, v3, off, off ; encoding: [0x63,0x01,0x00,0xf8,0x02,0x03,0x00,0x00] +0x63,0x01,0x00,0xf8,0x02,0x03,0x00,0x00 + +# GFX12: export mrtz v4, v3, v2, v1 row_en ; encoding: [0x8f,0x20,0x00,0xf8,0x04,0x03,0x02,0x01] +0x8f,0x20,0x00,0xf8,0x04,0x03,0x02,0x01 + +# GFX12: export mrtz v4, v3, off, off done row_en ; encoding: [0x83,0x28,0x00,0xf8,0x04,0x03,0x00,0x00] +0x83,0x28,0x00,0xf8,0x04,0x03,0x00,0x00 diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt index 7029f090faa4..c061462339b6 100644 --- a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_sop1.txt @@ -3219,6 +3219,9 @@ # GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_TBA) ; encoding: [0x85,0x4c,0x80,0xbe] 0x85,0x4c,0x80,0xbe +# GFX12: s_sendmsg_rtn_b32 s0, sendmsg(MSG_RTN_GET_SE_AID_ID) ; encoding: [0x86,0x4c,0x80,0xbe] +0x86,0x4c,0x80,0xbe + # GFX12: s_setpc_b64 s[0:1] ; encoding: [0x00,0x48,0x80,0xbe] 0x00,0x48,0x80,0xbe diff --git a/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt new file mode 100644 index 000000000000..bd1db69625ca --- /dev/null +++ b/llvm/test/MC/Disassembler/AMDGPU/gfx12_dasm_vinterp.txt @@ -0,0 +1,251 @@ +# RUN: llvm-mc -arch=amdgcn -mcpu=gfx1200 -disassemble %s | FileCheck -strict-whitespace -check-prefix=GFX12 %s + +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# Check that unused bits in the encoding are ignored. +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0xe0,0xcd,0x01,0x05,0x0e,0x1c + +# GFX12: v_interp_p10_f32 v1, v10, v20, v30 wait_exp:0{{$}} +0x01,0x00,0x00,0xcd,0x0a,0x29,0x7a,0x04 + +# GFX12: v_interp_p10_f32 v2, v11, v21, v31 wait_exp:0{{$}} +0x02,0x00,0x00,0xcd,0x0b,0x2b,0x7e,0x04 + +# GFX12: v_interp_p10_f32 v3, v12, v22, v32 wait_exp:0{{$}} +0x03,0x00,0x00,0xcd,0x0c,0x2d,0x82,0x04 + +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x24 + +# GFX12: v_interp_p10_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x44 + +# GFX12: v_interp_p10_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x00,0xcd,0x01,0x05,0x0e,0x84 + +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} +0x00,0x87,0x00,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f32 v1, v10, v20, v30 wait_exp:0{{$}} +0x01,0x00,0x01,0xcd,0x0a,0x29,0x7a,0x04 + +# GFX12: v_interp_p2_f32 v2, v11, v21, v31 wait_exp:0{{$}} +0x02,0x00,0x01,0xcd,0x0b,0x2b,0x7e,0x04 + +# GFX12: v_interp_p2_f32 v3, v12, v22, v32 wait_exp:0{{$}} +0x03,0x00,0x01,0xcd,0x0c,0x2d,0x82,0x04 + +# GFX12: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x24 + +# GFX12: v_interp_p2_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x44 + +# GFX12: v_interp_p2_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x01,0xcd,0x01,0x05,0x0e,0x84 + +# GFX12: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f32 v0, v1, v2, v3 clamp wait_exp:7{{$}} +0x00,0x87,0x01,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x24 + +# GFX12: v_interp_p10_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x44 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x02,0xcd,0x01,0x05,0x0e,0x84 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x02,0xcd,0x01,0x05,0x0e,0xe4 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x24 + +# GFX12: v_interp_p2_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x44 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x03,0xcd,0x01,0x05,0x0e,0x84 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x03,0xcd,0x01,0x05,0x0e,0xe4 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x24 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x44 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x04,0xcd,0x01,0x05,0x0e,0x84 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p10_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x04,0xcd,0x01,0x05,0x0e,0xe4 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, -v1, v2, v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x24 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, -v2, v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x44 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, -v3 wait_exp:0{{$}} +0x00,0x00,0x05,0xcd,0x01,0x05,0x0e,0x84 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp wait_exp:0{{$}} +0x00,0x80,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:1{{$}} +0x00,0x01,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 wait_exp:7{{$}} +0x00,0x07,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,0] wait_exp:0{{$}} +0x00,0x08,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,1,0,0] wait_exp:0{{$}} +0x00,0x10,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,1,0] wait_exp:0{{$}} +0x00,0x20,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[0,0,0,1] wait_exp:0{{$}} +0x00,0x40,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,1,1,1] wait_exp:0{{$}} +0x00,0x78,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0x4d,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, v1, v2, v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0x04 + +# GFX12: v_interp_p2_rtz_f16_f32 v0, -v1, -v2, -v3 clamp op_sel:[1,0,0,1] wait_exp:5{{$}} +0x00,0xcd,0x05,0xcd,0x01,0x05,0x0e,0xe4 diff --git a/llvm/test/MC/Disassembler/X86/apx/invpcid.txt b/llvm/test/MC/Disassembler/X86/apx/invpcid.txt new file mode 100644 index 000000000000..8987332e9c9d --- /dev/null +++ b/llvm/test/MC/Disassembler/X86/apx/invpcid.txt @@ -0,0 +1,6 @@ +# RUN: llvm-mc --disassemble %s -triple=x86_64 | FileCheck %s --check-prefixes=ATT +# RUN: llvm-mc --disassemble %s -triple=x86_64 -x86-asm-syntax=intel --output-asm-variant=1 | FileCheck %s --check-prefixes=INTEL + +# ATT: invpcid 291(%r28,%r29,4), %r19 +# INTEL: invpcid r19, xmmword ptr [r28 + 4*r29 + 291] +0x62,0x8c,0x7a,0x08,0xf2,0x9c,0xac,0x23,0x01,0x00,0x00 diff --git a/llvm/test/MC/ELF/reloc-directive.s b/llvm/test/MC/ELF/reloc-directive.s index 59d7ace40d7e..a4658f938d0d 100644 --- a/llvm/test/MC/ELF/reloc-directive.s +++ b/llvm/test/MC/ELF/reloc-directive.s @@ -10,11 +10,14 @@ # ASM-NEXT: .Ltmp2: # ASM-NEXT: .reloc 2+.Ltmp2, R_X86_64_NONE, foo # ASM-NEXT: .reloc (1+foo)+3, R_X86_64_NONE, data+1 +# ASM-NEXT: .Ltmp3: +# ASM-NEXT: .reloc .Ltmp3, BFD_RELOC_NONE, unused # CHECK: 0x2 R_X86_64_NONE foo 0x0 # CHECK-NEXT: 0x0 R_X86_64_NONE foo 0x0 # CHECK-NEXT: 0x3 R_X86_64_NONE foo 0x0 # CHECK-NEXT: 0x4 R_X86_64_NONE data 0x1 +# CHECK-NEXT: 0x1 R_X86_64_NONE unused 0x0 .text .globl foo @@ -24,6 +27,7 @@ foo: .reloc .-1, R_X86_64_NONE, foo .reloc 2+., R_X86_64_NONE, foo .reloc 1+foo+3, R_X86_64_NONE, data+1 + .reloc ., BFD_RELOC_NONE, unused .data .globl data diff --git a/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s index d2d0d44f639a..03179480147a 100644 --- a/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s +++ b/llvm/test/MC/PowerPC/ppc64-encoding-ISA31.s @@ -1044,3 +1044,301 @@ # CHECK-BE: hashchkp 5, -512(1) # encoding: [0x7c,0x01,0x2d,0x64] # CHECK-LE: hashchkp 5, -512(1) # encoding: [0x64,0x2d,0x01,0x7c] hashchkp 5, -512(1) + +## +## Extended Mnemonics +## + +# CHECK-BE: wait # encoding: [0x7c,0x00,0x00,0x3c] +# CHECK-LE: wait # encoding: [0x3c,0x00,0x00,0x7c] + wait +# CHECK-BE: wait # encoding: [0x7c,0x00,0x00,0x3c] +# CHECK-LE: wait # encoding: [0x3c,0x00,0x00,0x7c] + wait 0 +# CHECK-BE: waitrsv # encoding: [0x7c,0x20,0x00,0x3c] +# CHECK-LE: waitrsv # encoding: [0x3c,0x00,0x20,0x7c] + wait 1 +# CHECK-BE: waitrsv # encoding: [0x7c,0x20,0x00,0x3c] +# CHECK-LE: waitrsv # encoding: [0x3c,0x00,0x20,0x7c] + waitrsv +# CHECK-BE: wait 2 0 # encoding: [0x7c,0x40,0x00,0x3c] +# CHECK-LE: wait 2 0 # encoding: [0x3c,0x00,0x40,0x7c] + pause_short + +# CHECK-BE: sync # encoding: [0x7c,0x00,0x04,0xac] +# CHECK-LE: sync # encoding: [0xac,0x04,0x00,0x7c] + sync +# CHECK-BE: ptesync # encoding: [0x7c,0x40,0x04,0xac] +# CHECK-LE: ptesync # encoding: [0xac,0x04,0x40,0x7c] + sync 2 +# CHECK-BE: sync # encoding: [0x7c,0x00,0x04,0xac] +# CHECK-LE: sync # encoding: [0xac,0x04,0x00,0x7c] + hwsync +# CHECK-BE: sync 1 # encoding: [0x7c,0x20,0x04,0xac] +# CHECK-LE: sync 1 # encoding: [0xac,0x04,0x20,0x7c] + wsync +# CHECK-BE: ptesync # encoding: [0x7c,0x40,0x04,0xac] +# CHECK-LE: ptesync # encoding: [0xac,0x04,0x40,0x7c] + ptesync +# CHECK-BE: phwsync # encoding: [0x7c,0x80,0x04,0xac] +# CHECK-LE: phwsync # encoding: [0xac,0x04,0x80,0x7c] + phwsync +# CHECK-BE: plwsync # encoding: [0x7c,0xa0,0x04,0xac] +# CHECK-LE: plwsync # encoding: [0xac,0x04,0xa0,0x7c] + plwsync +# CHECK-BE: stncisync # encoding: [0x7c,0x21,0x04,0xac] +# CHECK-LE: stncisync # encoding: [0xac,0x04,0x21,0x7c] + stncisync +# CHECK-BE: stcisync # encoding: [0x7c,0x02,0x04,0xac] +# CHECK-LE: stcisync # encoding: [0xac,0x04,0x02,0x7c] + stcisync +# CHECK-BE: stsync # encoding: [0x7c,0x03,0x04,0xac] +# CHECK-LE: stsync # encoding: [0xac,0x04,0x03,0x7c] + stsync + +# CHECK-BE: paddi 4, 5, 4294965249 # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x38,0x85,0xf8,0x01] +# CHECK-LE: paddi 4, 5, 4294965249 # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0x85,0x38] + paddi 4, 5, 4294965249 +# CHECK-BE: pli 4, 4294965249 # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x38,0x80,0xf8,0x01] +# CHECK-LE: pli 4, 4294965249 # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0x80,0x38] + pli 4, 4294965249 +# CHECK-BE: paddi 4, 3, 4294965249 # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x38,0x83,0xf8,0x01] +# CHECK-LE: paddi 4, 3, 4294965249 # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0x83,0x38] + pla 4, 4294965249(3) +# CHECK-BE: paddi 4, 0, 4294965249, 1 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0x38,0x80,0xf8,0x01] +# CHECK-LE: paddi 4, 0, 4294965249, 1 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0x80,0x38] + pla 4, 4294965249 +# CHECK-BE: paddi 4, 5, -80, 0 # encoding: [0x06,0x03,0xff,0xff, +# CHECK-BE-SAME: 0x38,0x85,0xff,0xb0] +# CHECK-LE: paddi 4, 5, -80, 0 # encoding: [0xff,0xff,0x03,0x06, +# CHECK-LE-SAME: 0xb0,0xff,0x85,0x38] + psubi 4, 5, 80 + +# CHECK-BE: plbz 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x88,0xa3,0xf8,0x01] +# CHECK-LE: plbz 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0x88] + plbz 5, 4294965249(3) +# CHECK-BE: plbz 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0x88,0xa0,0xf8,0x01] +# CHECK-LE: plbz 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0x88] + plbz 5, 4294965249 +# CHECK-BE: plhz 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xa0,0xa3,0xf8,0x01] +# CHECK-LE: plhz 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xa0] + plhz 5, 4294965249(3) +# CHECK-BE: plhz 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xa0,0xa0,0xf8,0x01] +# CHECK-LE: plhz 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xa0] + plhz 5, 4294965249 +# CHECK-BE: plha 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xa8,0xa3,0xf8,0x01] +# CHECK-LE: plha 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xa8] + plha 5, 4294965249(3) +# CHECK-BE: plha 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xa8,0xa0,0xf8,0x01] +# CHECK-LE: plha 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xa8] + plha 5, 4294965249 +# CHECK-BE: plwz 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x80,0xa3,0xf8,0x01] +# CHECK-LE: plwz 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0x80] + plwz 5, 4294965249(3) +# CHECK-BE: plwz 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0x80,0xa0,0xf8,0x01] +# CHECK-LE: plwz 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0x80] + plwz 5, 4294965249 +# CHECK-BE: plwa 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xa4,0xa3,0xf8,0x01] +# CHECK-LE: plwa 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xa4] + plwa 5, 4294965249(3) +# CHECK-BE: plwa 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xa4,0xa0,0xf8,0x01] +# CHECK-LE: plwa 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xa4] + plwa 5, 4294965249 +# CHECK-BE: pld 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xe4,0xa3,0xf8,0x01] +# CHECK-LE: pld 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xe4] + pld 5, 4294965249(3) +# CHECK-BE: pld 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xe4,0xa0,0xf8,0x01] +# CHECK-LE: pld 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xe4] + pld 5, 4294965249 +# CHECK-BE: pstb 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x98,0xa3,0xf8,0x01] +# CHECK-LE: pstb 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0x98] + pstb 5, 4294965249(3) +# CHECK-BE: pstb 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0x98,0xa0,0xf8,0x01] +# CHECK-LE: pstb 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0x98] + pstb 5, 4294965249 +# CHECK-BE: psth 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xb0,0xa3,0xf8,0x01] +# CHECK-LE: psth 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xb0] + psth 5, 4294965249(3) +# CHECK-BE: psth 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xb0,0xa0,0xf8,0x01] +# CHECK-LE: psth 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xb0] + psth 5, 4294965249 +# CHECK-BE: pstw 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0x90,0xa3,0xf8,0x01] +# CHECK-LE: pstw 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0x90] + pstw 5, 4294965249(3) +# CHECK-BE: pstw 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0x90,0xa0,0xf8,0x01] +# CHECK-LE: pstw 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0x90] + pstw 5, 4294965249 +# CHECK-BE: pstd 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xf4,0xa3,0xf8,0x01] +# CHECK-LE: pstd 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xf4] + pstd 5, 4294965249(3) +# CHECK-BE: pstd 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xf4,0xa0,0xf8,0x01] +# CHECK-LE: pstd 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xf4] + pstd 5, 4294965249 +# CHECK-BE: plfs 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xc0,0xa3,0xf8,0x01] +# CHECK-LE: plfs 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xc0] + plfs 5, 4294965249(3) +# CHECK-BE: plfs 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xc0,0xa0,0xf8,0x01] +# CHECK-LE: plfs 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xc0] + plfs 5, 4294965249 +# CHECK-BE: plfd 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xc8,0xa3,0xf8,0x01] +# CHECK-LE: plfd 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xc8] + plfd 5, 4294965249(3) +# CHECK-BE: plfd 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xc8,0xa0,0xf8,0x01] +# CHECK-LE: plfd 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xc8] + plfd 5, 4294965249 +# CHECK-BE: pstfs 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xd0,0xa3,0xf8,0x01] +# CHECK-LE: pstfs 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xd0] + pstfs 5, 4294965249(3) +# CHECK-BE: pstfs 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xd0,0xa0,0xf8,0x01] +# CHECK-LE: pstfs 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xd0] + pstfs 5, 4294965249 +# CHECK-BE: pstfd 5, 4294965249(3) # encoding: [0x06,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xd8,0xa3,0xf8,0x01] +# CHECK-LE: pstfd 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xd8] + pstfd 5, 4294965249(3) +# CHECK-BE: pstfd 5, 4294965249 # encoding: [0x06,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xd8,0xa0,0xf8,0x01] +# CHECK-LE: pstfd 5, 4294965249 # encoding: [0xff,0xff,0x10,0x06, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xd8] + pstfd 5, 4294965249 +# CHECK-BE: plxsd 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xa8,0xa3,0xf8,0x01] +# CHECK-LE: plxsd 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xa8] + plxsd 5, 4294965249(3) +# CHECK-BE: plxsd 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xa8,0xa0,0xf8,0x01] +# CHECK-LE: plxsd 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xa8] + plxsd 5, 4294965249 +# CHECK-BE: plxssp 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xac,0xa3,0xf8,0x01] +# CHECK-LE: plxssp 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xac] + plxssp 5, 4294965249(3) +# CHECK-BE: plxssp 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xac,0xa0,0xf8,0x01] +# CHECK-LE: plxssp 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xac] + plxssp 5, 4294965249 +# CHECK-BE: plxv 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xc8,0xa3,0xf8,0x01] +# CHECK-LE: plxv 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xc8] + plxv 5, 4294965249(3) +# CHECK-BE: plxv 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xc8,0xa0,0xf8,0x01] +# CHECK-LE: plxv 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xc8] + plxv 5, 4294965249 +# CHECK-BE: plxvp 6, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xe8,0xc3,0xf8,0x01] +# CHECK-LE: plxvp 6, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xc3,0xe8] + plxvp 6, 4294965249(3) +# CHECK-BE: plxvp 6, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xe8,0xc0,0xf8,0x01] +# CHECK-LE: plxvp 6, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xc0,0xe8] + plxvp 6, 4294965249 +# CHECK-BE: pstxsd 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xb8,0xa3,0xf8,0x01] +# CHECK-LE: pstxsd 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xb8] + pstxsd 5, 4294965249(3) +# CHECK-BE: pstxsd 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xb8,0xa0,0xf8,0x01] +# CHECK-LE: pstxsd 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xb8] + pstxsd 5, 4294965249 +# CHECK-BE: pstxssp 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xbc,0xa3,0xf8,0x01] +# CHECK-LE: pstxssp 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xbc] + pstxssp 5, 4294965249(3) +# CHECK-BE: pstxssp 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xbc,0xa0,0xf8,0x01] +# CHECK-LE: pstxssp 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xbc] + pstxssp 5, 4294965249 +# CHECK-BE: pstxv 5, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xd8,0xa3,0xf8,0x01] +# CHECK-LE: pstxv 5, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa3,0xd8] + pstxv 5, 4294965249(3) +# CHECK-BE: pstxv 5, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xd8,0xa0,0xf8,0x01] +# CHECK-LE: pstxv 5, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xa0,0xd8] + pstxv 5, 4294965249 +# CHECK-BE: pstxvp 6, 4294965249(3) # encoding: [0x04,0x00,0xff,0xff, +# CHECK-BE-SAME: 0xf8,0xc3,0xf8,0x01] +# CHECK-LE: pstxvp 6, 4294965249(3) # encoding: [0xff,0xff,0x00,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xc3,0xf8] + pstxvp 6, 4294965249(3) +# CHECK-BE: pstxvp 6, 4294965249 # encoding: [0x04,0x10,0xff,0xff, +# CHECK-BE-SAME: 0xf8,0xc0,0xf8,0x01] +# CHECK-LE: pstxvp 6, 4294965249 # encoding: [0xff,0xff,0x10,0x04, +# CHECK-LE-SAME: 0x01,0xf8,0xc0,0xf8] + pstxvp 6, 4294965249 diff --git a/llvm/test/MC/PowerPC/ppc64-errors.s b/llvm/test/MC/PowerPC/ppc64-errors.s index 627ae410db88..1c03dc084acd 100644 --- a/llvm/test/MC/PowerPC/ppc64-errors.s +++ b/llvm/test/MC/PowerPC/ppc64-errors.s @@ -53,8 +53,8 @@ wait 4 # CHECK: error: invalid operand for instruction -# CHECK-NEXT: sync 4 - sync 4 +# CHECK-NEXT: sync 8 + sync 8 # Unsigned 3-bit immediate operands diff --git a/llvm/test/MC/X86/apx/invpcid-att.s b/llvm/test/MC/X86/apx/invpcid-att.s new file mode 100644 index 000000000000..a0e827ecadb0 --- /dev/null +++ b/llvm/test/MC/X86/apx/invpcid-att.s @@ -0,0 +1,8 @@ +# RUN: llvm-mc -triple x86_64 --show-encoding %s | FileCheck %s +# RUN: not llvm-mc -triple i386 -show-encoding %s 2>&1 | FileCheck %s --check-prefix=ERROR + +# ERROR-COUNT-1: error: +# ERROR-NOT: error: +# CHECK: invpcid 291(%r28,%r29,4), %r19 +# CHECK: encoding: [0x62,0x8c,0x7a,0x08,0xf2,0x9c,0xac,0x23,0x01,0x00,0x00] + invpcid 291(%r28,%r29,4), %r19 diff --git a/llvm/test/MC/X86/apx/invpcid-intel.s b/llvm/test/MC/X86/apx/invpcid-intel.s new file mode 100644 index 000000000000..e9993d26962e --- /dev/null +++ b/llvm/test/MC/X86/apx/invpcid-intel.s @@ -0,0 +1,5 @@ +# RUN: llvm-mc -triple x86_64 -x86-asm-syntax=intel -output-asm-variant=1 --show-encoding %s | FileCheck %s + +# CHECK: invpcid r19, xmmword ptr [r28 + 4*r29 + 291] +# CHECK: encoding: [0x62,0x8c,0x7a,0x08,0xf2,0x9c,0xac,0x23,0x01,0x00,0x00] + invpcid r19, xmmword ptr [r28 + 4*r29 + 291] diff --git a/llvm/test/MachineVerifier/test_g_prefetch.mir b/llvm/test/MachineVerifier/test_g_prefetch.mir new file mode 100644 index 000000000000..a08b0803fc35 --- /dev/null +++ b/llvm/test/MachineVerifier/test_g_prefetch.mir @@ -0,0 +1,40 @@ +# RUN: not --crash llc -o - -mtriple=aarch64 -global-isel -run-pass=none -verify-machineinstrs %s 2>&1 | FileCheck %s +# REQUIRES: aarch64-registered-target + +--- +name: test_fcmp +legalized: true +regBankSelected: false +selected: false +tracksRegLiveness: true +liveins: +body: | + bb.0: + liveins: $x0, $w0, $q0 + %s32:_(s32) = COPY $w0 + %ptr:_(p0) = COPY $x0 + + G_PREFETCH %ptr + ; CHECK: *** Bad machine code: Too few operands *** + ; CHECK: 4 operands expected, but 1 given. + + G_PREFETCH %ptr, 0, 0, 0, 0 + ; CHECK: *** Bad machine code: Extra explicit operand on non-variadic instruction *** + ; CHECK: operand 4: + + G_PREFETCH %s32, 0, 0, 0 + ; CHECK: *** Bad machine code: addr operand must be a pointer *** + ; CHECK: operand 0: + + G_PREFETCH %ptr, 10, 0, 0 + ; CHECK: *** Bad machine code: rw operand must be an immediate 0-1 *** + ; CHECK: operand 1: + + G_PREFETCH %ptr, 0, 10, 0 + ; CHECK: *** Bad machine code: locality operand must be an immediate 0-3 *** + ; CHECK: operand 2: + + G_PREFETCH %ptr, 0, 0, 10 + ; CHECK: *** Bad machine code: cache type operand must be an immediate 0-1 *** + ; CHECK: operand 3: +... diff --git a/llvm/test/TableGen/ContextlessPredicates.td b/llvm/test/TableGen/ContextlessPredicates.td index 7f081e9a0ec0..0f4c4d0c450e 100644 --- a/llvm/test/TableGen/ContextlessPredicates.td +++ b/llvm/test/TableGen/ContextlessPredicates.td @@ -26,7 +26,7 @@ def : Pat<(test_atomic_op_frag GPR32:$ptr, GPR32:$val) , // CHECK_NOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK_NOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ATOMICRMW_XCHG, // CHECK_NOPT-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/4, -// CHECK_NOPT-NEXT: // MIs[0] dst +// CHECK_NOPT-NEXT: // MIs[0] DstI[dst] // CHECK_NOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK_NOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // CHECK_NOPT-NEXT: // MIs[0] ptr diff --git a/llvm/test/TableGen/DefaultOpsGlobalISel.td b/llvm/test/TableGen/DefaultOpsGlobalISel.td index 13ee2631ecb0..c997467c570a 100644 --- a/llvm/test/TableGen/DefaultOpsGlobalISel.td +++ b/llvm/test/TableGen/DefaultOpsGlobalISel.td @@ -35,7 +35,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/0, GICP_gi_SelectSrcMods, // CHECK: GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/1, GICP_gi_SelectSrcMods, // CHECK: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FMAX, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // mods0 // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0 // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/1, /*SubOperand*/1, // mods1 @@ -48,7 +48,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckComplexPattern, /*MI*/0, /*Op*/1, /*Renderer*/0, GICP_gi_SelectClampOMod, // CHECK: // (ffloor:{ *:[f32] } (SelectClampOMod:{ *:[f32] } f32:{ *:[f32] }:$src0, omod:{ *:[i32] }:$omod, i1:{ *:[i1] }:$clamp)) => (FLOMP:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp, omod:{ *:[i32] }:$omod) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FLOMP, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0 // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/2, // clamp // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // omod @@ -56,7 +56,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FCANONICALIZE, // CHECK: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FMAX, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // mods // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // mods @@ -68,7 +68,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FCOS, // CHECK: // (fcos:{ *:[f32] } (SelectOMod:{ *:[f32] } f32:{ *:[f32] }:$src0, i32:{ *:[i32] }:$omod)) => (FLAMP:{ *:[f32] } FPR32:{ *:[f32] }:$src0, omod:{ *:[i32] }:$omod) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FLAMP, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0 // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // omod // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0, @@ -84,7 +84,7 @@ def clamp : OperandWithDefaultOps ; // CHECK-NEXT: GIR_AddImm, /*InsnID*/1, /*Imm*/0, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FEEPLE, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0 // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // clamp @@ -95,7 +95,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FSIN, // CHECK: // (fsin:{ *:[f32] } (SelectClamp:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp)) => (FFOO:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FFOO, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0 // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // clamp // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -104,7 +104,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FSQRT, // CHECK: // (fsqrt:{ *:[f32] } (SelectClamp:{ *:[f32] } f32:{ *:[f32] }:$src0, i1:{ *:[i1] }:$clamp)) => (FLAMP:{ *:[f32] } FPR32:{ *:[f32] }:$src0, 93:{ *:[i32] }, clamp:{ *:[i1] }:$clamp) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FLAMP, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src0 // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/93, // CHECK-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // clamp @@ -113,7 +113,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_ROUND, // CHECK: // (fround:{ *:[f32] } f32:{ *:[f32] }:$src0) => (FBAR:{ *:[f32] } f32:{ *:[f32] }:$src0) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FBAR, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0 // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -121,7 +121,7 @@ def clamp : OperandWithDefaultOps ; // CHECK: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC_TRUNC, // CHECK: // (ftrunc:{ *:[f32] } f32:{ *:[f32] }:$src0) => (FFOO:{ *:[f32] } FPR32:{ *:[f32] }:$src0) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FFOO, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0 // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td index 2d968977701f..6ae1305aa1aa 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/builtins/match-table-replacerreg.td @@ -28,11 +28,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const int64_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static int64_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/65, 180, /*)*//*default:*//*Label 2*/ 192, -// CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ 120, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_FNEG*//*Label 1*/ 165, -// CHECK-NEXT: // Label 0: @120 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ 164, // Rule ID 1 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/65, 181, /*)*//*default:*//*Label 2*/ 193, +// CHECK-NEXT: /*TargetOpcode::G_UNMERGE_VALUES*//*Label 0*/ 121, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_FNEG*//*Label 1*/ 166, +// CHECK-NEXT: // Label 0: @121 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ 165, // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled, // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: // MIs[0] a @@ -57,10 +57,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ReplaceRegWithTempReg, /*OldInsnID*/0, /*OldOpIdx*/1, /*TempRegID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 3: @164 +// CHECK-NEXT: // Label 3: @165 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @165 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ 191, // Rule ID 0 // +// CHECK-NEXT: // Label 1: @166 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ 192, // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled, // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: // No operand predicates @@ -75,9 +75,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_ReplaceReg, /*OldInsnID*/0, /*OldOpIdx*/0, /*NewInsnId*/1, /*NewOpIdx*/1, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @191 +// CHECK-NEXT: // Label 4: @192 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @192 +// CHECK-NEXT: // Label 2: @193 // CHECK-NEXT: GIM_Reject, // CHECK-NEXT: }; // CHECK-NEXT: return MatchTable0; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td index 0495a66a7c57..fd5f7db0b4f1 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-imms.td @@ -34,12 +34,12 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const int64_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static int64_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/19, 126, /*)*//*default:*//*Label 3*/ 194, -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_CONSTANT*//*Label 1*/ 138, 0, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 2*/ 165, -// CHECK-NEXT: // Label 0: @112 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ 137, // Rule ID 0 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/19, 127, /*)*//*default:*//*Label 3*/ 195, +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_CONSTANT*//*Label 1*/ 139, 0, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 2*/ 166, +// CHECK-NEXT: // Label 0: @113 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ 138, // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled, // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] a @@ -51,10 +51,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 4: @137 +// CHECK-NEXT: // Label 4: @138 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @138 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ 164, // Rule ID 2 // +// CHECK-NEXT: // Label 1: @139 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 5*/ 165, // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule2Enabled, // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] a @@ -66,10 +66,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddCImm, /*InsnID*/0, /*Type*/GILLT_s32, /*Imm*/42, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 5: @164 +// CHECK-NEXT: // Label 5: @165 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @165 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ 193, // Rule ID 1 // +// CHECK-NEXT: // Label 2: @166 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 6*/ 194, // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled, // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -83,9 +83,9 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 6: @193 +// CHECK-NEXT: // Label 6: @194 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @194 +// CHECK-NEXT: // Label 3: @195 // CHECK-NEXT: GIM_Reject, // CHECK-NEXT: }; // CHECK-NEXT: return MatchTable0; diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td index 5cb9206ca5f2..b62ebcf24687 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-patfrag-root.td @@ -28,7 +28,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK: const int64_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static int64_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/118, 181, /*)*//*default:*//*Label 3*/ 152, +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/119, 182, /*)*//*default:*//*Label 3*/ 152, // CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 0*/ 68, 0, 0, 0, 0, 0, 0, // CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 1*/ 93, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // CHECK-NEXT: /*TargetOpcode::G_FPEXT*//*Label 2*/ 127, diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td index a74f7fbbe1cc..6777089f846f 100644 --- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td +++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table.td @@ -132,15 +132,15 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // Verify match table. // CHECK: const int64_t *GenMyCombiner::getMatchTable() const { // CHECK-NEXT: constexpr static int64_t MatchTable0[] = { -// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/19, 126, /*)*//*default:*//*Label 6*/ 267, -// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ 112, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ 141, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ 181, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ 216, 0, 0, 0, 0, -// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ 231, 0, -// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ 239, -// CHECK-NEXT: // Label 0: @112 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ 133, // Rule ID 4 // +// CHECK-NEXT: GIM_SwitchOpcode, /*MI*/0, /*[*/19, 127, /*)*//*default:*//*Label 6*/ 268, +// CHECK-NEXT: /*TargetOpcode::COPY*//*Label 0*/ 113, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_AND*//*Label 1*/ 142, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_STORE*//*Label 2*/ 182, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_TRUNC*//*Label 3*/ 217, 0, 0, 0, 0, +// CHECK-NEXT: /*TargetOpcode::G_SEXT*//*Label 4*/ 232, 0, +// CHECK-NEXT: /*TargetOpcode::G_ZEXT*//*Label 5*/ 240, +// CHECK-NEXT: // Label 0: @113 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 7*/ 134, // Rule ID 4 // // CHECK-NEXT: GIM_CheckFeatures, GIFBS_HasAnswerToEverything, // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule3Enabled, // CHECK-NEXT: // MIs[0] a @@ -155,8 +155,8 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #3: InstTest1 // CHECK-NEXT: GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 7: @133 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ 140, // Rule ID 3 // +// CHECK-NEXT: // Label 7: @134 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 8*/ 141, // Rule ID 3 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule2Enabled, // CHECK-NEXT: // MIs[0] a // CHECK-NEXT: // No operand predicates @@ -165,10 +165,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: // Combiner Rule #2: InstTest0 // CHECK-NEXT: GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner1, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 8: @140 +// CHECK-NEXT: // Label 8: @141 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 1: @141 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ 180, // Rule ID 6 // +// CHECK-NEXT: // Label 1: @142 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 9*/ 181, // Rule ID 6 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule5Enabled, // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, // CHECK-NEXT: // MIs[0] dst @@ -186,10 +186,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // z // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 9: @180 +// CHECK-NEXT: // Label 9: @181 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 2: @181 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ 215, // Rule ID 5 // +// CHECK-NEXT: // Label 2: @182 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 10*/ 216, // Rule ID 5 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule4Enabled, // CHECK-NEXT: // MIs[0] tmp // CHECK-NEXT: GIM_RecordInsnIgnoreCopies, /*DefineMI*/1, /*MI*/0, /*OpIdx*/0, // MIs[1] @@ -207,32 +207,32 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner2, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 10: @215 +// CHECK-NEXT: // Label 10: @216 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 3: @216 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ 223, // Rule ID 0 // +// CHECK-NEXT: // Label 3: @217 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 11*/ 224, // Rule ID 0 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule0Enabled, // CHECK-NEXT: // Combiner Rule #0: WipOpcodeTest0; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 11: @223 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ 230, // Rule ID 1 // +// CHECK-NEXT: // Label 11: @224 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 12*/ 231, // Rule ID 1 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled, // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_TRUNC' // CHECK-NEXT: GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 12: @230 +// CHECK-NEXT: // Label 12: @231 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 4: @231 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ 238, // Rule ID 2 // +// CHECK-NEXT: // Label 4: @232 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 13*/ 239, // Rule ID 2 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule1Enabled, // CHECK-NEXT: // Combiner Rule #1: WipOpcodeTest1; wip_match_opcode 'G_SEXT' // CHECK-NEXT: GIR_CustomAction, GICXXCustomAction_CombineApplyGICombiner0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 13: @238 +// CHECK-NEXT: // Label 13: @239 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 5: @239 -// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ 266, // Rule ID 7 // +// CHECK-NEXT: // Label 5: @240 +// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 14*/ 267, // Rule ID 7 // // CHECK-NEXT: GIM_CheckSimplePredicate, GICXXPred_Simple_IsRule6Enabled, // CHECK-NEXT: // MIs[0] dst // CHECK-NEXT: // No operand predicates @@ -247,10 +247,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [ // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_Done, -// CHECK-NEXT: // Label 14: @266 +// CHECK-NEXT: // Label 14: @267 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: // Label 6: @267 +// CHECK-NEXT: // Label 6: @268 // CHECK-NEXT: GIM_Reject, -// CHECK-NEXT: }; +// CHECK-NEXT: }; // Size: 2152 bytes // CHECK-NEXT: return MatchTable0; // CHECK-NEXT: } diff --git a/llvm/test/TableGen/GlobalISelEmitter-input-discard.td b/llvm/test/TableGen/GlobalISelEmitter-input-discard.td index 6d0d1de5d06c..68a9553a8b73 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-input-discard.td +++ b/llvm/test/TableGen/GlobalISelEmitter-input-discard.td @@ -21,7 +21,7 @@ def FOO : I<(outs GPR32:$dst), (ins GPR32Op:$src0, GPR32Op:$src1), []>; // GISEL-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define, // GISEL-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::FOO, -// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // GISEL-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/3, // src1 // GISEL-NEXT: GIR_EraseFromParent, /*InsnID*/0, diff --git a/llvm/test/TableGen/GlobalISelEmitter-multiple-output-discard.td b/llvm/test/TableGen/GlobalISelEmitter-multiple-output-discard.td index 5f442b067d9c..64cf31f3772e 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-multiple-output-discard.td +++ b/llvm/test/TableGen/GlobalISelEmitter-multiple-output-discard.td @@ -22,10 +22,10 @@ def : GINodeEquiv; def : Pat<(two_out GPR32:$val), (THREE_OUTS GPR32:$val)>; // CHECK: GIM_CheckOpcode, /*MI*/0, MyTarget::G_TWO_OUT, -// CHECK-NEXT: // MIs[0] out1 +// CHECK-NEXT: // MIs[0] DstI[out1] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, -// CHECK-NEXT: // MIs[0] out2 +// CHECK-NEXT: // MIs[0] DstI[out2] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // MIs[0] val @@ -34,8 +34,8 @@ def : Pat<(two_out GPR32:$val), (THREE_OUTS GPR32:$val)>; // CHECK-NEXT: // (two_out:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$val) => (THREE_OUTS:{ *:[i32] }:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$val) // CHECK-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s32, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::THREE_OUTS, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // out1 -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // out2 +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[out1] +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // DstI[out2] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/RegState::Define|RegState::Dead, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // val // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, diff --git a/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td b/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td index c98ac73c61af..85ac3ace0364 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td +++ b/llvm/test/TableGen/GlobalISelEmitter-multiple-output.td @@ -30,10 +30,10 @@ def : Pat<(loadpost (p0 GPR32:$addr), (i32 GPR32:$off)), >; // CHECK: GIM_CheckOpcode, /*MI*/0, MyTarget::G_POST_LOAD, -// CHECK-NEXT: // MIs[0] val +// CHECK-NEXT: // MIs[0] DstI[val] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, -// CHECK-NEXT: // MIs[0] ptr_out +// CHECK-NEXT: // MIs[0] DstI[ptr_out] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_p0s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // MIs[0] addr @@ -64,10 +64,10 @@ def : GINodeEquiv; def : Pat<(two_in GPR32:$i1, GPR32:$i2), (TWO_INS GPR32:$i2, GPR32:$i1)>; // CHECK: GIM_CheckOpcode, /*MI*/0, MyTarget::G_TWO_IN, -// CHECK-NEXT: // MIs[0] out1 +// CHECK-NEXT: // MIs[0] DstI[out1] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, -// CHECK-NEXT: // MIs[0] out2 +// CHECK-NEXT: // MIs[0] DstI[out2] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // MIs[0] i1 @@ -78,8 +78,8 @@ def : Pat<(two_in GPR32:$i1, GPR32:$i2), (TWO_INS GPR32:$i2, GPR32:$i1)>; // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/3, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // (two_in:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$i1, GPR32:{ *:[i32] }:$i2) => (TWO_INS:{ *:[i32] }:{ *:[i32] } GPR32:{ *:[i32] }:$i2, GPR32:{ *:[i32] }:$i1) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::TWO_INS, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // out1 -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // out2 +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[out1] +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // DstI[out2] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/3, // i2 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // i1 // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, diff --git a/llvm/test/TableGen/GlobalISelEmitter-nested-subregs.td b/llvm/test/TableGen/GlobalISelEmitter-nested-subregs.td index c48c82aa142a..61f79bc04b27 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-nested-subregs.td +++ b/llvm/test/TableGen/GlobalISelEmitter-nested-subregs.td @@ -32,7 +32,7 @@ def A0 : RegisterClass<"MyTarget", [i32], 32, (add a0)>; // CHECK: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ANYEXT, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s16, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::A0RegClassID, // CHECK-NEXT: // MIs[0] src @@ -52,7 +52,7 @@ def A0 : RegisterClass<"MyTarget", [i32], 32, (add a0)>; // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, MyTarget::A0RegClassID, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/2, MyTarget::A0bRegClassID, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempSubRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, MyTarget::lo16, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, MyTarget::A0wRegClassID, diff --git a/llvm/test/TableGen/GlobalISelEmitter-output-discard.td b/llvm/test/TableGen/GlobalISelEmitter-output-discard.td index c755d8377e61..1cfe49dee097 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-output-discard.td +++ b/llvm/test/TableGen/GlobalISelEmitter-output-discard.td @@ -15,7 +15,7 @@ def ADD_CO : I<(outs GPR32:$dst, GPR8:$flag), // GISEL-NEXT: // (add:{ *:[i32] } i32:{ *:[i32] }:$src0, i32:{ *:[i32] }:$src1) => (ADD_CO:{ *:[i32] }:{ *:[i8] } GPR32:{ *:[i32] }:$src0, GPR32:{ *:[i32] }:$src1) // GISEL-NEXT: GIR_MakeTempReg, /*TempRegID*/0, /*TypeID*/GILLT_s8, // GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::ADD_CO, -// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // GISEL-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/RegState::Define|RegState::Dead, // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0 // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src1 diff --git a/llvm/test/TableGen/GlobalISelEmitter-zero-reg.td b/llvm/test/TableGen/GlobalISelEmitter-zero-reg.td index 374430bc427e..63ad5d5cd03d 100644 --- a/llvm/test/TableGen/GlobalISelEmitter-zero-reg.td +++ b/llvm/test/TableGen/GlobalISelEmitter-zero-reg.td @@ -24,7 +24,7 @@ def INST : PredI<(outs GPR32:$dst), (ins GPR32:$src), []>; // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // MIs[0] src @@ -32,7 +32,7 @@ def INST : PredI<(outs GPR32:$dst), (ins GPR32:$src), []>; // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, // CHECK-NEXT: // (ld:{ *:[i32] } GPR32:{ *:[i32] }:$src)<><> => (INST:{ *:[i32] } GPR32:{ *:[i32] }:$src) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INST, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src // CHECK-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::NoRegister, /*AddRegisterRegFlags*/0, // CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*MergeInsnID's*/0, GIU_MergeMemOperands_EndOfList, diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td index b7a81894f644..562fdefcd3d6 100644 --- a/llvm/test/TableGen/GlobalISelEmitter.td +++ b/llvm/test/TableGen/GlobalISelEmitter.td @@ -247,7 +247,7 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; } // R19O-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, // R19N-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/4, // R19N-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SELECT, -// R19N-NEXT: // MIs[0] dst +// R19N-NEXT: // MIs[0] DstI[dst] // R19N-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // R19N-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // R19N-NEXT: // MIs[0] src1 @@ -290,7 +290,7 @@ def HasC : Predicate<"Subtarget->hasC()"> { let RecomputePerFunction = 1; } // R19C-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/1, /*RendererID*/2, /*SubOperand*/1, // src5b // R19C-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // R19C-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN3, -// R19C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// R19C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // R19C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // R19C-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/1, // src2b // R19C-NEXT: GIR_ComplexSubOperandRenderer, /*InsnID*/0, /*RendererID*/0, /*SubOperand*/0, // src2a @@ -341,7 +341,7 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), // R21O-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/1, /*RC*/MyTarget::GPR32RegClassID, // R21N-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/4, // R21N-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SELECT, -// R21N-NEXT: // MIs[0] dst +// R21N-NEXT: // MIs[0] DstI[dst] // R21N-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // R21N-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // R21N-NEXT: // MIs[0] src1 @@ -359,7 +359,7 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), // R21C-NEXT: // (select:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2, complex:{ *:[i32] }:$src3)<> => (INSN2:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src3, complex:{ *:[i32] }:$src2) // R21C-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN2, -// R21C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// R21C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // R21C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // R21C-NEXT: GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/1, // R21C-NEXT: GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/0, @@ -397,7 +397,7 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), // // R20N-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // R20N-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SUB, -// R20N-NEXT: // MIs[0] dst +// R20N-NEXT: // MIs[0] DstI[dst] // R20N-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // R20N-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // R20N-NEXT: // MIs[0] src1 @@ -410,7 +410,7 @@ def : Pat<(select GPR32:$src1, (complex_rr GPR32:$src2a, GPR32:$src2b), // R20C-NEXT: GIM_CheckComplexPattern, /*MI*/0, /*Op*/2, /*Renderer*/0, GICP_gi_complex, // R20C-NEXT: // (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2) => (INSN1:{ *:[i32] } GPR32:{ *:[i32] }:$src1, complex:{ *:[i32] }:$src2) // R20C-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSN1, -// R20C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// R20C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // R20C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // R20C-NEXT: GIR_ComplexRenderer, /*InsnID*/0, /*RendererID*/0, // R20C-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -462,7 +462,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3), // R00C-NEXT: GIM_CheckFeatures, GIFBS_HasA, // R00N-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // R00N-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SUB, -// R00N-NEXT: // MIs[0] dst +// R00N-NEXT: // MIs[0] DstI[dst] // R00N-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // R00N-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // R00N-NEXT: // MIs[0] Operand 1 @@ -501,7 +501,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3), // R00C-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/2, // R00C-NEXT: // (sub:{ *:[i32] } (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src3, GPR32:{ *:[i32] }:$src4)) => (INSNBOB:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3, GPR32:{ *:[i32] }:$src4) // R00C-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::INSNBOB, -// R00C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// R00C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // R00C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1 // R00C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // src2 // R00C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/1, // src3 @@ -517,7 +517,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3), // R00O-NEXT: GIM_Reject, // R00O: // Label [[DEFAULT_NUM]]: @[[DEFAULT]] // R00O-NEXT: GIM_Reject, -// R00O-NEXT: }; +// R00O-NEXT: }; // Size: 9888 bytes def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4), [(set GPR32:$dst, @@ -544,7 +544,7 @@ def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, G // R01O-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // // R01N-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_INTRINSIC, -// R01N-NEXT: // MIs[0] dst +// R01N-NEXT: // MIs[0] DstI[dst] // R01N-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // R01N-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // R01N-NEXT: // MIs[0] Operand 1 @@ -555,7 +555,7 @@ def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, G // R01C-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID, // R01C-NEXT: // (intrinsic_wo_chain:{ *:[i32] } [[ID:[0-9]+]]:{ *:[iPTR] }, GPR32:{ *:[i32] }:$src1) => (MOV:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // R01C-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOV, -// R01C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// R01C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // R01C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src1 // R01C-NEXT: GIR_EraseFromParent, /*InsnID*/0, // R01C-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -591,7 +591,7 @@ def MOV : I<(outs GPR32:$dst), (ins GPR32:$src1), // // R02N-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // R02N-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR, -// R02N-NEXT: // MIs[0] dst +// R02N-NEXT: // MIs[0] DstI[dst] // R02N-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // R02N-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // R02N-NEXT: // MIs[0] src1 @@ -603,7 +603,7 @@ def MOV : I<(outs GPR32:$dst), (ins GPR32:$src1), // R02C-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -2 // R02C-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -2:{ *:[i32] }) => (XORI:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // R02C-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XORI, -// R02C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// R02C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // R02C-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/-1, // R02C-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // R02C-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -624,7 +624,7 @@ def XORI : I<(outs GPR32:$dst), (ins m1:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -635,7 +635,7 @@ def XORI : I<(outs GPR32:$dst), (ins m1:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -3 // NOOPT-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -3:{ *:[i32] }) => (XOR:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XOR, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::R0, // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -653,7 +653,7 @@ def XOR : I<(outs GPR32:$dst), (ins Z:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -664,7 +664,7 @@ def XOR : I<(outs GPR32:$dst), (ins Z:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -4 // NOOPT-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -4:{ *:[i32] }) => (XORlike:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XORlike, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/-1, // NOOPT-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::R0, // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 @@ -683,7 +683,7 @@ def XORlike : I<(outs GPR32:$dst), (ins m1Z:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -694,7 +694,7 @@ def XORlike : I<(outs GPR32:$dst), (ins m1Z:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -5, // NOOPT-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$src1, -5:{ *:[i32] }) => (XORManyDefaults:{ *:[i32] } GPR32:{ *:[i32] }:$src1) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::XORManyDefaults, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/-1, // NOOPT-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::R0, // NOOPT-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::R0, @@ -717,7 +717,7 @@ def XORManyDefaults : I<(outs GPR32:$dst), (ins m1Z:$src3, Z:$src2, GPR32:$src1) // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_XOR, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Wm @@ -728,7 +728,7 @@ def XORManyDefaults : I<(outs GPR32:$dst), (ins m1Z:$src3, Z:$src2, GPR32:$src1) // NOOPT-NEXT: GIM_CheckConstantInt, /*MI*/0, /*Op*/2, -1, // NOOPT-NEXT: // (xor:{ *:[i32] } GPR32:{ *:[i32] }:$Wm, -1:{ *:[i32] }) => (ORN:{ *:[i32] } R0:{ *:[i32] }, GPR32:{ *:[i32] }:$Wm) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::ORN, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_AddRegister, /*InsnID*/0, MyTarget::R0, // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // Wm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -746,7 +746,7 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>; // NOOPT-NEXT: GIM_CheckFeatures, GIFBS_HasA, // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 @@ -768,7 +768,7 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>; // NOOPT-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, // NOOPT-NEXT: // (mul:{ *:[i32] } (add:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), GPR32:{ *:[i32] }:$src3) => (MULADD:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MULADD, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1 // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // src2 // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src3 @@ -784,7 +784,7 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>; // NOOPT-NEXT: GIM_CheckFeatures, GIFBS_HasA, // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src3 @@ -806,13 +806,13 @@ def : Pat<(not GPR32:$Wm), (ORN R0, GPR32:$Wm)>; // NOOPT-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, // NOOPT-NEXT: // (mul:{ *:[i32] } GPR32:{ *:[i32] }:$src3, (add:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2)) => (MULADD:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src3) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MULADD, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src1 // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // src2 // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src3 // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 26, +// NOOPT-NEXT: // GIR_Coverage, 27, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -826,14 +826,14 @@ def MULADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 // NOOPT-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, 1, // NOOPT-NEXT: // 1:{ *:[i32] } => (MOV1:{ *:[i32] }) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOV1, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, // NOOPT-NEXT: // GIR_Coverage, 7, @@ -848,14 +848,14 @@ def MOV1 : I<(outs GPR32:$dst), (ins), [(set GPR32:$dst, 1)]>; // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT, // NOOPT-NEXT: GIM_CheckI64ImmPredicate, /*MI*/0, /*Predicate*/GICXXPred_I64_Predicate_simm8, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 // NOOPT-NEXT: // No operand predicates // NOOPT-NEXT: // (imm:{ *:[i32] })<>:$imm => (MOVimm8:{ *:[i32] } (imm:{ *:[i32] }):$imm) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVimm8, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -872,14 +872,14 @@ def MOVimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, simm8:$i // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT, // NOOPT-NEXT: GIM_CheckAPIntImmPredicate, /*MI*/0, /*Predicate*/GICXXPred_APInt_Predicate_simm9, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 // NOOPT-NEXT: // No operand predicates // NOOPT-NEXT: // (imm:{ *:[i32] })<>:$imm => (MOVimm9:{ *:[i32] } (imm:{ *:[i32] }):$imm) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVimm9, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -896,14 +896,14 @@ def MOVimm9 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, simm9:$i // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT, // NOOPT-NEXT: GIM_CheckI64ImmPredicate, /*MI*/0, /*Predicate*/GICXXPred_I64_Predicate_cimm8, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 // NOOPT-NEXT: // No operand predicates // NOOPT-NEXT: // (imm:{ *:[i32] })<><>:$imm => (MOVcimm8:{ *:[i32] } (cimm8_xform:{ *:[i32] } (imm:{ *:[i32] }):$imm)) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVcimm8, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_CustomRenderer, /*InsnID*/0, /*OldInsnID*/0, /*Renderer*/GICR_renderImm, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -919,14 +919,14 @@ def MOVcimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, cimm8:$ // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_FCONSTANT, // NOOPT-NEXT: GIM_CheckAPFloatImmPredicate, /*MI*/0, /*Predicate*/GICXXPred_APFloat_Predicate_fpimmz, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::FPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 // NOOPT-NEXT: // No operand predicates // NOOPT-NEXT: // (fpimm:{ *:[f32] })<>:$imm => (MOVfpimmz:{ *:[f32] } (fpimm:{ *:[f32] }):$imm) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVfpimmz, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_CopyFConstantAsFPImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -941,7 +941,7 @@ def MOVcimm8 : I<(outs GPR32:$dst), (ins i32imm:$imm), [(set GPR32:$dst, cimm8:$ // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // NOOPT-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // NOOPT-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -964,7 +964,7 @@ def LOAD : I<(outs GPR32:$dst), (ins GPR32:$src1), // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // NOOPT-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // NOOPT-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_p0s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src @@ -987,7 +987,7 @@ def : Pat<(load GPR32:$src), // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SEXTLOAD, // NOOPT-NEXT: GIM_CheckMemorySizeEqualTo, /*MI*/0, /*MMO*/0, /*Size*/2, // NOOPT-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -1008,7 +1008,7 @@ def SEXTLOAD : I<(outs GPR32:$dst), (ins GPR32:$src1), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -1032,7 +1032,7 @@ def ADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src{{$}} @@ -1042,7 +1042,7 @@ def ADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2), // NOOPT-NEXT: GIM_CheckIsSameOperand, /*MI*/0, /*OpIdx*/2, /*OtherMI*/0, /*OtherOpIdx*/1, // NOOPT-NEXT: // (add:{ *:[i32] } GPR32:{ *:[i32] }:$src, GPR32:{ *:[i32] }:$src) => (DOUBLE:{ *:[i32] } GPR32:{ *:[i32] }:$src) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::DOUBLE, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -1052,12 +1052,41 @@ def ADD : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2), def DOUBLE : I<(outs GPR32:$dst), (ins GPR32:$src), [(set GPR32:$dst, (add GPR32:$src, GPR32:$src))]>; +//===- Test a pattern with unintended operand name clash. ----------------===// + +// Check that using the same name for +// - Def operand of the instruction corresponding to the root node of the +// pattern's destination +// - one of operands in the pattern itself +// does not introduce unexpected GIM_CheckIsSameOperand predicate. + +// NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], +// NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, +// NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD, +// NOOPT-NEXT: // MIs[0] DstI[samename] +// NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, +// NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, +// NOOPT-NEXT: // MIs[0] samename +// NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/1, /*Type*/GILLT_s32, +// NOOPT-NEXT: // MIs[0] othername +// NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/2, /*Type*/GILLT_s32, +// NOOPT-NEXT: // (add:{ *:[i32] } i32:{ *:[i32] }:$samename, i32:{ *:[i32] }:$othername) => (InsnWithSpeciallyNamedDef:{ *:[i32] } i32:{ *:[i32] }:$samename, i32:{ *:[i32] }:$othername) +// NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::InsnWithSpeciallyNamedDef, +// NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, +// NOOPT-NEXT: // GIR_Coverage, 24, +// NOOPT-NEXT: GIR_Done, +// NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] + +def InsnWithSpeciallyNamedDef : I<(outs GPR32:$samename), (ins GPR32:$src1, GPR32:$src2), []>; +def : Pat<(add i32:$samename, i32:$othername), + (InsnWithSpeciallyNamedDef i32:$samename, i32:$othername)>; + //===- Test a simple pattern with ValueType operands. ----------------------===// // // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -1067,7 +1096,7 @@ def DOUBLE : I<(outs GPR32:$dst), (ins GPR32:$src), [(set GPR32:$dst, (add GPR32 // NOOPT-NEXT: // (add:{ *:[i32] } i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) => (ADD:{ *:[i32] } i32:{ *:[i32] }:$src1, i32:{ *:[i32] }:$src2) // NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/MyTarget::ADD, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, -// NOOPT-NEXT: // GIR_Coverage, 24, +// NOOPT-NEXT: // GIR_Coverage, 25, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1080,7 +1109,7 @@ def : Pat<(add i32:$src1, i32:$src2), // NOOPT-NEXT: GIM_CheckFeatures, GIFBS_HasA_HasB_HasC, // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -1091,7 +1120,7 @@ def : Pat<(add i32:$src1, i32:$src2), // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/2, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // (mul:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) => (MUL:{ *:[i32] } GPR32:{ *:[i32] }:$src2, GPR32:{ *:[i32] }:$src1) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MUL, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/2, // src2 // NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src1 // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -1110,7 +1139,7 @@ def MUL : I<(outs GPR32:$dst), (ins GPR32:$src2, GPR32:$src1), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_BITCAST, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] src1 @@ -1119,7 +1148,7 @@ def MUL : I<(outs GPR32:$dst), (ins GPR32:$src2, GPR32:$src1), // NOOPT-NEXT: // (bitconvert:{ *:[i32] } FPR32:{ *:[f32] }:$src1) => (COPY_TO_REGCLASS:{ *:[i32] } FPR32:{ *:[f32] }:$src1, GPR32:{ *:[i32] }) // NOOPT-NEXT: GIR_MutateOpcode, /*InsnID*/0, /*RecycleInsnID*/0, /*Opcode*/TargetOpcode::COPY, // NOOPT-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, MyTarget::GPR32RegClassID, -// NOOPT-NEXT: // GIR_Coverage, 25, +// NOOPT-NEXT: // GIR_Coverage, 26, // NOOPT-NEXT: GIR_Done, // NOOPT-NEXT: // Label [[LABEL_NUM]]: @[[LABEL]] @@ -1131,14 +1160,14 @@ def : Pat<(i32 (bitconvert FPR32:$src1)), // NOOPT-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ [[LABEL:[0-9]+]], // NOOPT-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // NOOPT-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_CONSTANT, -// NOOPT-NEXT: // MIs[0] dst +// NOOPT-NEXT: // MIs[0] DstI[dst] // NOOPT-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // NOOPT-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // NOOPT-NEXT: // MIs[0] Operand 1 // NOOPT-NEXT: // No operand predicates // NOOPT-NEXT: // (imm:{ *:[i32] }):$imm => (MOVimm:{ *:[i32] } (imm:{ *:[i32] }):$imm) // NOOPT-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MOVimm, -// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// NOOPT-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // NOOPT-NEXT: GIR_CopyConstantAsSImm, /*NewInsnID*/0, /*OldInsnID*/0, // imm // NOOPT-NEXT: GIR_EraseFromParent, /*InsnID*/0, // NOOPT-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -1169,5 +1198,5 @@ def BR : I<(outs), (ins unknown:$target), [(br bb:$target)]>; // NOOPT-NEXT: GIM_Reject, -// NOOPT-NEXT: }; +// NOOPT-NEXT: }; // Size: 11408 bytes // NOOPT-NEXT: return MatchTable0; diff --git a/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td b/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td index d07ef4e300ee..5a75cb129eec 100644 --- a/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td +++ b/llvm/test/TableGen/GlobalISelEmitterCustomPredicate.td @@ -76,7 +76,7 @@ def and_or_pat : PatFrag< // CHECK: GIM_Try, /*On fail goto*//*Label 0*/ 99, // Rule ID 7 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] src2 @@ -106,7 +106,7 @@ def and_or_pat : PatFrag< // CHECK: GIM_Try, /*On fail goto*//*Label 1*/ 198, // Rule ID 3 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_AND, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] Operand 1 @@ -151,7 +151,7 @@ def mul_pat : PatFrag< // CHECK: GIM_Try, /*On fail goto*//*Label 2*/ 293, // Rule ID 4 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] Operand 1 @@ -180,7 +180,7 @@ def mul_pat : PatFrag< // CHECK: GIM_Try, /*On fail goto*//*Label 3*/ 388, // Rule ID 8 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] src2 @@ -226,7 +226,7 @@ def sub3_pat : PatFrag< // CHECK: GIM_Try, /*On fail goto*//*Label 4*/ 475, // Rule ID 0 // // CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SUB, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] Operand 1 diff --git a/llvm/test/TableGen/GlobalISelEmitterHwModes.td b/llvm/test/TableGen/GlobalISelEmitterHwModes.td index b185feaf009f..04f33648313b 100644 --- a/llvm/test/TableGen/GlobalISelEmitterHwModes.td +++ b/llvm/test/TableGen/GlobalISelEmitterHwModes.td @@ -133,7 +133,7 @@ class I Pat> // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s64, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPRRegClassID, // CHECK-NEXT: // MIs[0] src1 @@ -151,7 +151,7 @@ class I Pat> // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPRRegClassID, // CHECK-NEXT: // MIs[0] src1 @@ -175,7 +175,7 @@ def LOAD : I<(outs GPR:$dst), (ins GPR:$src1), // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_p0s64, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPRRegClassID, // CHECK-NEXT: // MIs[0] src @@ -193,7 +193,7 @@ def LOAD : I<(outs GPR:$dst), (ins GPR:$src1), // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_LOAD, // CHECK-NEXT: GIM_CheckMemorySizeEqualToLLT, /*MI*/0, /*MMO*/0, /*OpIdx*/0, // CHECK-NEXT: GIM_CheckAtomicOrdering, /*MI*/0, /*Order*/(int64_t)AtomicOrdering::NotAtomic, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_p0s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPRRegClassID, // CHECK-NEXT: // MIs[0] src diff --git a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizer.td b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizer.td index cd7a177b8426..6a369b6a7b88 100644 --- a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizer.td +++ b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizer.td @@ -61,7 +61,7 @@ def LOAD16Imm : I<(outs GPR16:$dst), (ins GPR16:$src), []>; // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/1, // CHECK-NEXT: // (ld:{ *:[i16] } (add:{ *:[i16] } GPR16:{ *:[i16] }:$src, 10:{ *:[i16] }))<><> => (LOAD16Imm:{ *:[i16] } GPR16:{ *:[i16] }:$src) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::LOAD16Imm, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/1, // src // CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/0, /*MergeInsnID's*/0, 1, GIU_MergeMemOperands_EndOfList, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, diff --git a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand-invalid.td b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand-invalid.td index 44e1b08bdd82..2423c3bd32d5 100644 --- a/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand-invalid.td +++ b/llvm/test/TableGen/GlobalISelEmitterMatchTableOptimizerSameOperand-invalid.td @@ -33,7 +33,7 @@ def InstThreeOperands : I<(outs GPR32:$dst), (ins GPR32:$cond, GPR32:$src,GPR32: // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/2, // CHECK-NEXT: // (select:{ *:[i32] } (setcc:{ *:[i32] } GPR32:{ *:[i32] }:$cond, 0:{ *:[i32] }, SETEQ:{ *:[Other] }), (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), GPR32:{ *:[i32] }:$src2) => (InstThreeOperands:{ *:[i32] } GPR32:{ *:[i32] }:$cond, GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::InstThreeOperands, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // cond // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/1, // src1 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/2, // src2 @@ -61,7 +61,7 @@ def InstThreeOperands : I<(outs GPR32:$dst), (ins GPR32:$cond, GPR32:$src,GPR32: // CHECK-NEXT: GIM_CheckIsSafeToFold, /*InsnID*/2, // CHECK-NEXT: // (select:{ *:[i32] } (setcc:{ *:[i32] } GPR32:{ *:[i32] }:$cond, 0:{ *:[i32] }, SETNE:{ *:[Other] }), (sub:{ *:[i32] } GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2), GPR32:{ *:[i32] }:$src2) => (InstThreeOperands:{ *:[i32] } GPR32:{ *:[i32] }:$cond, GPR32:{ *:[i32] }:$src1, GPR32:{ *:[i32] }:$src2) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::InstThreeOperands, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/1, /*OpIdx*/2, // cond // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/1, // src1 // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/2, /*OpIdx*/2, // src2 diff --git a/llvm/test/TableGen/GlobalISelEmitterRegSequence.td b/llvm/test/TableGen/GlobalISelEmitterRegSequence.td index 1b7391497f12..34783e688399 100644 --- a/llvm/test/TableGen/GlobalISelEmitterRegSequence.td +++ b/llvm/test/TableGen/GlobalISelEmitterRegSequence.td @@ -32,7 +32,7 @@ def SUBSOME_INSN : I<(outs SRegs:$dst), (ins SOP:$src), []>; // CHECK: GIM_CheckNumOperands, /*MI*/0, /*Expected*/2, // CHECK-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_SEXT, -// CHECK-NEXT: // MIs[0] dst +// CHECK-NEXT: // MIs[0] DstI[dst] // CHECK-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // CHECK-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/Test::DRegsRegClassID, // CHECK-NEXT: // MIs[0] src @@ -50,7 +50,7 @@ def SUBSOME_INSN : I<(outs SRegs:$dst), (ins SOP:$src), []>; // CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // src // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::REG_SEQUENCE, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*SubRegIndex*/1, // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/1, /*TempRegFlags*/0, diff --git a/llvm/test/TableGen/GlobalISelEmitterSubreg.td b/llvm/test/TableGen/GlobalISelEmitterSubreg.td index c8405735aaea..af3db1590368 100644 --- a/llvm/test/TableGen/GlobalISelEmitterSubreg.td +++ b/llvm/test/TableGen/GlobalISelEmitterSubreg.td @@ -72,7 +72,7 @@ def : Pat<(sub (complex DOP:$src1, DOP:$src2), 77), // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, Test::SRegsRegClassID, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, Test::DRegsRegClassID, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SOME_INSN2, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/1, /*TempRegFlags*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, @@ -87,7 +87,7 @@ def : Pat<(i32 (anyext i16:$src)), (INSERT_SUBREG (i32 (IMPLICIT_DEF)), SOP:$src // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::INSERT_SUBREG, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/1, @@ -115,7 +115,7 @@ def : Pat<(i32 (anyext i16:$src)), (SOME_INSN (INSERT_SUBREG (i32 (IMPLICIT_DEF) // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, Test::DRegsRegClassID, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/2, Test::SRegsRegClassID, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SOME_INSN, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -145,7 +145,7 @@ def : Pat<(i32 (anyext i16:$src)), (INSERT_SUBREG (i32 (IMPLICIT_DEF)), (SUBSOME // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/1, /*TempRegID*/0, /*TempRegFlags*/RegState::Define, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::INSERT_SUBREG, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/1, /*TempRegFlags*/0, // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/1, @@ -182,7 +182,7 @@ def : Pat<(i16 (trunc (bitreverse DOP:$src))), // CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/1, /*OpIdx*/1, // src // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempSubRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, sub0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, Test::SRegsRegClassID, @@ -207,7 +207,7 @@ def : Pat<(i16 (trunc (bitreverse DOP:$src))), // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/0, Test::SRegsRegClassID, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/1, /*Op*/1, Test::DRegsRegClassID, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::SUBSOME_INSN2, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -219,7 +219,7 @@ def : Pat<(i16 (trunc DOP:$src)), (EXTRACT_SUBREG DOP:$src, sub0)>; // CHECK-LABEL: // (trunc:{ *:[i16] } DOP:{ *:[i32] }:$src) => (EXTRACT_SUBREG:{ *:[i16] } DOP:{ *:[i32] }:$src, sub0:{ *:[i32] }) // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::COPY, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_CopySubReg, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, /*SubRegIdx*/1, // src // CHECK-NEXT: GIR_EraseFromParent, /*InsnID*/0, // CHECK-NEXT: GIR_ConstrainOperandRC, /*InsnID*/0, /*Op*/0, Test::SRegsRegClassID, @@ -236,7 +236,7 @@ def : Pat<(i32 (zext SOP:$src)), // CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/1, // src // CHECK-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/1, // CHECK-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/TargetOpcode::SUBREG_TO_REG, -// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// CHECK-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/0, // CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/0, // CHECK-NEXT: GIR_AddImm, /*InsnID*/0, /*Imm*/1, diff --git a/llvm/test/TableGen/gisel-physreg-input.td b/llvm/test/TableGen/gisel-physreg-input.td index e700141c203e..3dbd57bc45b7 100644 --- a/llvm/test/TableGen/gisel-physreg-input.td +++ b/llvm/test/TableGen/gisel-physreg-input.td @@ -27,7 +27,7 @@ class I Pat> // GISEL: GIM_Try, // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_ADD, -// GISEL-NEXT: // MIs[0] dst +// GISEL-NEXT: // MIs[0] DstI[dst] // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // GISEL-NEXT: // MIs[0] src0 @@ -41,7 +41,7 @@ class I Pat> // GISEL-NEXT: GIR_AddRegister, /*InsnID*/1, MyTarget::SPECIAL, /*AddRegisterRegFlags*/RegState::Define, // GISEL-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/2, // SPECIAL // GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::ADD_PHYS, -// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // src0 // GISEL-NEXT: GIR_EraseFromParent, /*InsnID*/0, // GISEL-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, @@ -55,7 +55,7 @@ def ADD_PHYS : I<(outs GPR32:$dst), (ins GPR32:$src0), // GISEL: GIM_Try, // GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3, // GISEL-NEXT: GIM_CheckOpcode, /*MI*/0, TargetOpcode::G_MUL, -// GISEL-NEXT: // MIs[0] dst +// GISEL-NEXT: // MIs[0] DstI[dst] // GISEL-NEXT: GIM_CheckType, /*MI*/0, /*Op*/0, /*Type*/GILLT_s32, // GISEL-NEXT: GIM_CheckRegBankForClass, /*MI*/0, /*Op*/0, /*RC*/MyTarget::GPR32RegClassID, // GISEL-NEXT: // MIs[0] SPECIAL @@ -69,7 +69,7 @@ def ADD_PHYS : I<(outs GPR32:$dst), (ins GPR32:$src0), // GISEL-NEXT: GIR_AddRegister, /*InsnID*/1, MyTarget::SPECIAL, /*AddRegisterRegFlags*/RegState::Define, // GISEL-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/2, // SPECIAL // GISEL-NEXT: GIR_BuildMI, /*InsnID*/0, /*Opcode*/MyTarget::MUL_PHYS, -// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // dst +// GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/0, // DstI[dst] // GISEL-NEXT: GIR_Copy, /*NewInsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, // SPECIAL // GISEL-NEXT: GIR_EraseFromParent, /*InsnID*/0, // GISEL-NEXT: GIR_ConstrainSelectedInstOperands, /*InsnID*/0, diff --git a/llvm/test/Transforms/AlignmentFromAssumptions/alignment-from-assumptions-track-users.ll b/llvm/test/Transforms/AlignmentFromAssumptions/alignment-from-assumptions-track-users.ll new file mode 100644 index 000000000000..1c388528ae4b --- /dev/null +++ b/llvm/test/Transforms/AlignmentFromAssumptions/alignment-from-assumptions-track-users.ll @@ -0,0 +1,204 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes=alignment-from-assumptions -S | FileCheck %s + +define void @widget(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(3) nocapture %arg1) { +; CHECK-LABEL: define void @widget( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(3) nocapture [[ARG1:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[ARG]], i64 1 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[GETELEMENTPTR]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(1) [[GETELEMENTPTR]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR2:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[ARG1]], i32 1 +; CHECK-NEXT: store i32 [[LOAD]], ptr addrspace(3) [[GETELEMENTPTR2]], align 4 +; CHECK-NEXT: ret void +; +bb: + %getelementptr = getelementptr inbounds i32, ptr addrspace(1) %arg, i64 1 + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %getelementptr, i64 4) ] + %load = load i32, ptr addrspace(1) %getelementptr, align 2 + %getelementptr2 = getelementptr inbounds i32, ptr addrspace(3) %arg1, i32 1 + store i32 %load, ptr addrspace(3) %getelementptr2, align 4 + ret void +} + +define void @wibble(ptr addrspace(1) nocapture readonly %arg, i32 %arg2, ptr addrspace(3) nocapture %arg3) { +; CHECK-LABEL: define void @wibble( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) nocapture [[ARG3:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[ICMP:%.*]] = icmp ugt i32 [[ARG2]], 10 +; CHECK-NEXT: br i1 [[ICMP]], label [[BB4:%.*]], label [[BB5:%.*]] +; CHECK: bb4: +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 6 +; CHECK-NEXT: br label [[BB7:%.*]] +; CHECK: bb5: +; CHECK-NEXT: [[GETELEMENTPTR6:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 7 +; CHECK-NEXT: br label [[BB7]] +; CHECK: bb7: +; CHECK-NEXT: [[PHI:%.*]] = phi ptr addrspace(1) [ [[GETELEMENTPTR]], [[BB4]] ], [ [[GETELEMENTPTR6]], [[BB5]] ] +; CHECK-NEXT: [[GETELEMENTPTR8:%.*]] = getelementptr inbounds i32, ptr addrspace(1) [[PHI]], i64 4 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[ARG]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(1) [[GETELEMENTPTR8]], align 2 +; CHECK-NEXT: [[GETELEMENTPTR9:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[ARG3]], i32 1 +; CHECK-NEXT: store i32 [[LOAD]], ptr addrspace(3) [[GETELEMENTPTR9]], align 4 +; CHECK-NEXT: ret void +; +bb: + %icmp = icmp ugt i32 %arg2, 10 + br i1 %icmp, label %bb4, label %bb5 + +bb4: ; preds = %bb + %getelementptr = getelementptr i32, ptr addrspace(1) %arg, i32 6 + br label %bb7 + +bb5: ; preds = %bb + %getelementptr6 = getelementptr i32, ptr addrspace(1) %arg, i32 7 + br label %bb7 + +bb7: ; preds = %bb5, %bb4 + %phi = phi ptr addrspace(1) [ %getelementptr, %bb4 ], [ %getelementptr6, %bb5 ] + %getelementptr8 = getelementptr inbounds i32, ptr addrspace(1) %phi, i64 4 + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %arg, i64 4) ] + %load = load i32, ptr addrspace(1) %getelementptr8, align 2 + %getelementptr9 = getelementptr inbounds i32, ptr addrspace(3) %arg3, i32 1 + store i32 %load, ptr addrspace(3) %getelementptr9, align 4 + ret void +} + +define void @ham(ptr addrspace(1) nocapture readonly %arg, i32 %arg2, ptr addrspace(3) nocapture %arg3) { +; CHECK-LABEL: define void @ham( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) nocapture [[ARG3:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 0 +; CHECK-NEXT: [[GETELEMENTPTR4:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 10 +; CHECK-NEXT: [[ICMP:%.*]] = icmp ugt i32 [[ARG2]], 10 +; CHECK-NEXT: br i1 [[ICMP]], label [[BB5:%.*]], label [[BB10:%.*]] +; CHECK: bb5: +; CHECK-NEXT: [[PHI:%.*]] = phi ptr addrspace(1) [ [[GETELEMENTPTR]], [[BB:%.*]] ], [ [[GETELEMENTPTR8:%.*]], [[BB5]] ] +; CHECK-NEXT: [[PHI6:%.*]] = phi i32 [ 0, [[BB]] ], [ [[ADD:%.*]], [[BB5]] ] +; CHECK-NEXT: [[GETELEMENTPTR7:%.*]] = getelementptr i32, ptr addrspace(1) [[PHI]], i32 4 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[ARG]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(1) [[GETELEMENTPTR7]], align 4 +; CHECK-NEXT: [[ADD]] = add i32 [[PHI6]], [[LOAD]] +; CHECK-NEXT: [[GETELEMENTPTR8]] = getelementptr i32, ptr addrspace(1) [[PHI]], i32 [[ARG2]] +; CHECK-NEXT: [[ICMP9:%.*]] = icmp eq ptr addrspace(1) [[GETELEMENTPTR8]], [[GETELEMENTPTR4]] +; CHECK-NEXT: br i1 [[ICMP9]], label [[BB5]], label [[BB10]] +; CHECK: bb10: +; CHECK-NEXT: [[PHI11:%.*]] = phi i32 [ 0, [[BB]] ], [ [[ADD]], [[BB5]] ] +; CHECK-NEXT: [[GETELEMENTPTR12:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[ARG3]], i32 1 +; CHECK-NEXT: store i32 [[PHI11]], ptr addrspace(3) [[GETELEMENTPTR12]], align 4 +; CHECK-NEXT: ret void +; +bb: + %getelementptr = getelementptr i32, ptr addrspace(1) %arg, i32 0 + %getelementptr4 = getelementptr i32, ptr addrspace(1) %arg, i32 10 + %icmp = icmp ugt i32 %arg2, 10 + br i1 %icmp, label %bb5, label %bb10 + +bb5: ; preds = %bb5, %bb + %phi = phi ptr addrspace(1) [ %getelementptr, %bb ], [ %getelementptr8, %bb5 ] + %phi6 = phi i32 [ 0, %bb ], [ %add, %bb5 ] + %getelementptr7 = getelementptr i32, ptr addrspace(1) %phi, i32 4 + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %arg, i64 4) ] + %load = load i32, ptr addrspace(1) %getelementptr7, align 2 + %add = add i32 %phi6, %load + %getelementptr8 = getelementptr i32, ptr addrspace(1) %phi, i32 %arg2 + %icmp9 = icmp eq ptr addrspace(1) %getelementptr8, %getelementptr4 + br i1 %icmp9, label %bb5, label %bb10 + +bb10: ; preds = %bb5, %bb + %phi11 = phi i32 [ 0, %bb ], [ %add, %bb5 ] + %getelementptr12 = getelementptr inbounds i32, ptr addrspace(3) %arg3, i32 1 + store i32 %phi11, ptr addrspace(3) %getelementptr12, align 4 + ret void +} + +define void @quux(ptr addrspace(1) nocapture readonly %arg, i32 %arg2, ptr addrspace(3) nocapture %arg3) { +; CHECK-LABEL: define void @quux( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) nocapture [[ARG3:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[ICMP:%.*]] = icmp ugt i32 [[ARG2]], 10 +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 6 +; CHECK-NEXT: [[GETELEMENTPTR4:%.*]] = getelementptr i32, ptr addrspace(1) [[ARG]], i32 7 +; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[ICMP]], ptr addrspace(1) [[GETELEMENTPTR]], ptr addrspace(1) [[GETELEMENTPTR4]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[ARG]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(1) [[SELECT]], align 2 +; CHECK-NEXT: [[GETELEMENTPTR5:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[ARG3]], i32 1 +; CHECK-NEXT: store i32 [[LOAD]], ptr addrspace(3) [[GETELEMENTPTR5]], align 4 +; CHECK-NEXT: ret void +; +bb: + %icmp = icmp ugt i32 %arg2, 10 + %getelementptr = getelementptr i32, ptr addrspace(1) %arg, i32 6 + %getelementptr4 = getelementptr i32, ptr addrspace(1) %arg, i32 7 + %select = select i1 %icmp, ptr addrspace(1) %getelementptr, ptr addrspace(1) %getelementptr4 + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %arg, i64 4) ] + %load = load i32, ptr addrspace(1) %select, align 2 + %getelementptr5 = getelementptr inbounds i32, ptr addrspace(3) %arg3, i32 1 + store i32 %load, ptr addrspace(3) %getelementptr5, align 4 + ret void +} + +define void @widget.1(ptr addrspace(1) nocapture readonly %arg, i32 %arg2, ptr addrspace(3) nocapture %arg3) { +; CHECK-LABEL: define void @widget.1( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG2:%.*]], ptr addrspace(3) nocapture [[ARG3:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[ADDRSPACECAST:%.*]] = addrspacecast ptr addrspace(3) [[ARG3]] to ptr addrspace(1) +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr i32, ptr addrspace(1) [[ADDRSPACECAST]] +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(3) [[ARG3]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load i32, ptr addrspace(1) [[GETELEMENTPTR]], align 2 +; CHECK-NEXT: [[GETELEMENTPTR4:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[ARG3]], i32 1 +; CHECK-NEXT: store i32 [[LOAD]], ptr addrspace(3) [[GETELEMENTPTR4]], align 4 +; CHECK-NEXT: ret void +; +bb: + %addrspacecast = addrspacecast ptr addrspace(3) %arg3 to ptr addrspace(1) + %getelementptr = getelementptr i32, ptr addrspace(1) %addrspacecast + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(3) %arg3, i64 4) ] + %load = load i32, ptr addrspace(1) %getelementptr, align 2 + %getelementptr4 = getelementptr inbounds i32, ptr addrspace(3) %arg3, i32 1 + store i32 %load, ptr addrspace(3) %getelementptr4, align 2 + ret void +} + +define void @baz(ptr addrspace(1) nocapture readonly %arg, ptr addrspace(3) nocapture %arg1) { +; CHECK-LABEL: define void @baz( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], ptr addrspace(3) nocapture [[ARG1:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr ptr addrspace(1), ptr addrspace(1) [[ARG]], i64 16 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[ARG]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load ptr addrspace(1), ptr addrspace(1) [[GETELEMENTPTR]], align 4 +; CHECK-NEXT: [[GETELEMENTPTR2:%.*]] = getelementptr inbounds i32, ptr addrspace(3) [[ARG1]], i32 1 +; CHECK-NEXT: store ptr addrspace(1) [[LOAD]], ptr addrspace(3) [[GETELEMENTPTR2]], align 2 +; CHECK-NEXT: ret void +; +bb: + %getelementptr = getelementptr ptr addrspace(1), ptr addrspace(1) %arg, i64 16 + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %arg, i64 4) ] + %load = load ptr addrspace(1), ptr addrspace(1) %getelementptr, align 2 + %getelementptr2 = getelementptr inbounds i32, ptr addrspace(3) %arg1, i32 1 + store ptr addrspace(1) %load, ptr addrspace(3) %getelementptr2, align 2 + ret void +} + +define void @foo(ptr addrspace(1) nocapture readonly %arg, i32 %arg1) { +; CHECK-LABEL: define void @foo( +; CHECK-SAME: ptr addrspace(1) nocapture readonly [[ARG:%.*]], i32 [[ARG1:%.*]]) { +; CHECK-NEXT: bb: +; CHECK-NEXT: [[GETELEMENTPTR:%.*]] = getelementptr ptr addrspace(3), ptr addrspace(1) [[ARG]], i64 16 +; CHECK-NEXT: call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) [[ARG]], i64 4) ] +; CHECK-NEXT: [[LOAD:%.*]] = load ptr addrspace(3), ptr addrspace(1) [[GETELEMENTPTR]], align 4 +; CHECK-NEXT: store i32 [[ARG1]], ptr addrspace(3) [[LOAD]], align 2 +; CHECK-NEXT: ret void +; +bb: + %getelementptr = getelementptr ptr addrspace(3), ptr addrspace(1) %arg, i64 16 + call void @llvm.assume(i1 true) [ "align"(ptr addrspace(1) %arg, i64 4) ] + %load = load ptr addrspace(3), ptr addrspace(1) %getelementptr, align 2 + store i32 %arg1, ptr addrspace(3) %load, align 2 + ret void +} + +; Function Attrs: nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) +declare void @llvm.assume(i1 noundef) #0 + +attributes #0 = { nocallback nofree nosync nounwind willreturn memory(inaccessiblemem: write) } diff --git a/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll b/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll index c28fc59014f5..58dff360ff6a 100644 --- a/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll +++ b/llvm/test/Transforms/IRCE/non-loop-invariant-rhs-instr.ll @@ -9,7 +9,7 @@ define i32 @test_01(i32 %A, i64 %Len, ptr %array) { ; CHECK-NEXT: br i1 [[TRIPCHECK]], label [[LOOP_PREHEADER:%.*]], label [[ZERO:%.*]] ; CHECK: loop.preheader: ; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[A:%.*]] to i64 -; CHECK-NEXT: [[TMP1:%.*]] = add i64 [[TMP0]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1 ; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[LEN]], i64 0) ; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[LEN]], [[SMIN]] ; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP2]], i64 [[TMP1]]) diff --git a/llvm/test/Transforms/IndVarSimplify/promote-iv-to-eliminate-casts.ll b/llvm/test/Transforms/IndVarSimplify/promote-iv-to-eliminate-casts.ll index 60e014b0efca..3f0ada281b1e 100644 --- a/llvm/test/Transforms/IndVarSimplify/promote-iv-to-eliminate-casts.ll +++ b/llvm/test/Transforms/IndVarSimplify/promote-iv-to-eliminate-casts.ll @@ -182,7 +182,7 @@ define void @promote_latch_condition_decrementing_loop_01(ptr %p, ptr %a) { ; CHECK-LABEL: @promote_latch_condition_decrementing_loop_01( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[LEN:%.*]] = load i32, ptr [[P:%.*]], align 4, !range [[RNG0:![0-9]+]] -; CHECK-NEXT: [[LEN_MINUS_1:%.*]] = add i32 [[LEN]], -1 +; CHECK-NEXT: [[LEN_MINUS_1:%.*]] = add nsw i32 [[LEN]], -1 ; CHECK-NEXT: [[ZERO_CHECK:%.*]] = icmp eq i32 [[LEN]], 0 ; CHECK-NEXT: br i1 [[ZERO_CHECK]], label [[LOOPEXIT:%.*]], label [[PREHEADER:%.*]] ; CHECK: preheader: diff --git a/llvm/test/Transforms/InstCombine/alloca.ll b/llvm/test/Transforms/InstCombine/alloca.ll index a64de28ee839..4247d6a30193 100644 --- a/llvm/test/Transforms/InstCombine/alloca.ll +++ b/llvm/test/Transforms/InstCombine/alloca.ll @@ -248,3 +248,14 @@ entry: call void (...) @use(ptr nonnull @int) [ "blah"(ptr %y) ] ret void } + +define void @test_inalloca_with_element_count(ptr %a) { +; ALL-LABEL: @test_inalloca_with_element_count( +; ALL-NEXT: [[ALLOCA1:%.*]] = alloca inalloca [10 x %struct_type], align 4 +; ALL-NEXT: call void @test9_aux(ptr nonnull inalloca([[STRUCT_TYPE:%.*]]) [[ALLOCA1]]) +; ALL-NEXT: ret void +; + %alloca = alloca inalloca %struct_type, i32 10, align 4 + call void @test9_aux(ptr inalloca(%struct_type) %alloca) + ret void +} diff --git a/llvm/test/Transforms/InstCombine/binop-of-displaced-shifts.ll b/llvm/test/Transforms/InstCombine/binop-of-displaced-shifts.ll index c86dfde6ddec..27a3c8743368 100644 --- a/llvm/test/Transforms/InstCombine/binop-of-displaced-shifts.ll +++ b/llvm/test/Transforms/InstCombine/binop-of-displaced-shifts.ll @@ -328,3 +328,32 @@ define <2 x i8> @shl_or_non_splat_out_of_range(<2 x i8> %x) { %binop = or <2 x i8> %shift, %shift2 ret <2 x i8> %binop } + +define i8 @shl_or_with_or_disjoint_instead_of_add(i8 %x) { +; CHECK-LABEL: define i8 @shl_or_with_or_disjoint_instead_of_add +; CHECK-SAME: (i8 [[X:%.*]]) { +; CHECK-NEXT: [[BINOP:%.*]] = shl i8 22, [[X]] +; CHECK-NEXT: ret i8 [[BINOP]] +; + %shift = shl i8 16, %x + %add = or disjoint i8 %x, 1 + %shift2 = shl i8 3, %add + %binop = or i8 %shift, %shift2 + ret i8 %binop +} + +define i8 @shl_or_with_or_instead_of_add(i8 %x) { +; CHECK-LABEL: define i8 @shl_or_with_or_instead_of_add +; CHECK-SAME: (i8 [[X:%.*]]) { +; CHECK-NEXT: [[SHIFT:%.*]] = shl i8 16, [[X]] +; CHECK-NEXT: [[ADD:%.*]] = or i8 [[X]], 1 +; CHECK-NEXT: [[SHIFT2:%.*]] = shl i8 3, [[ADD]] +; CHECK-NEXT: [[BINOP:%.*]] = or i8 [[SHIFT]], [[SHIFT2]] +; CHECK-NEXT: ret i8 [[BINOP]] +; + %shift = shl i8 16, %x + %add = or i8 %x, 1 + %shift2 = shl i8 3, %add + %binop = or i8 %shift, %shift2 + ret i8 %binop +} diff --git a/llvm/test/Transforms/InstCombine/ctlz-cttz-shifts.ll b/llvm/test/Transforms/InstCombine/ctlz-cttz-shifts.ll new file mode 100644 index 000000000000..86fef51872b1 --- /dev/null +++ b/llvm/test/Transforms/InstCombine/ctlz-cttz-shifts.ll @@ -0,0 +1,234 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -passes=instcombine -S < %s | FileCheck %s + +declare i32 @llvm.ctlz.i32(i32, i1) +declare i32 @llvm.cttz.i32(i32, i1) +declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1) +declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1) + +define i32 @lshr_ctlz_true(i32) { +; CHECK-LABEL: define i32 @lshr_ctlz_true( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTLZ:%.*]] = add i32 [[TMP0]], 9 +; CHECK-NEXT: ret i32 [[CTLZ]] +; + %lshr = lshr i32 8387584, %0 + %ctlz = call i32 @llvm.ctlz.i32(i32 %lshr, i1 true) + ret i32 %ctlz +} + +define i32 @shl_nuw_ctlz_true(i32) { +; CHECK-LABEL: define i32 @shl_nuw_ctlz_true( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTLZ:%.*]] = sub i32 9, [[TMP0]] +; CHECK-NEXT: ret i32 [[CTLZ]] +; + %shl = shl nuw i32 8387584, %0 + %ctlz = call i32 @llvm.ctlz.i32(i32 %shl, i1 true) + ret i32 %ctlz +} + +define i32 @shl_nuw_nsw_ctlz_true(i32) { +; CHECK-LABEL: define i32 @shl_nuw_nsw_ctlz_true( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTLZ:%.*]] = sub i32 9, [[TMP0]] +; CHECK-NEXT: ret i32 [[CTLZ]] +; + %shl = shl nuw nsw i32 8387584, %0 + %ctlz = call i32 @llvm.ctlz.i32(i32 %shl, i1 true) + ret i32 %ctlz +} + +define i32 @lshr_exact_cttz_true(i32) { +; CHECK-LABEL: define i32 @lshr_exact_cttz_true( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTTZ:%.*]] = sub i32 10, [[TMP0]] +; CHECK-NEXT: ret i32 [[CTTZ]] +; + %lshr = lshr exact i32 8387584, %0 + %cttz = call i32 @llvm.cttz.i32(i32 %lshr, i1 true) + ret i32 %cttz +} + +define i32 @shl_cttz_true(i32) { +; CHECK-LABEL: define i32 @shl_cttz_true( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTTZ:%.*]] = add i32 [[TMP0]], 10 +; CHECK-NEXT: ret i32 [[CTTZ]] +; + %shl = shl i32 8387584, %0 + %cttz = call i32 @llvm.cttz.i32(i32 %shl, i1 true) + ret i32 %cttz +} + +define <2 x i32> @vec2_lshr_ctlz_true(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_lshr_ctlz_true( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTLZ:%.*]] = add <2 x i32> [[TMP0]], +; CHECK-NEXT: ret <2 x i32> [[CTLZ]] +; + %div = lshr <2 x i32> , %0 + %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %div, i1 true) + ret <2 x i32> %ctlz +} + +define <2 x i32> @vec2_shl_nuw_ctlz_true(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_shl_nuw_ctlz_true( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTLZ:%.*]] = sub <2 x i32> , [[TMP0]] +; CHECK-NEXT: ret <2 x i32> [[CTLZ]] +; + %shl = shl nuw <2 x i32> , %0 + %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %shl, i1 true) + ret <2 x i32> %ctlz +} + +define <2 x i32> @vec2_shl_nuw_nsw_ctlz_true(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_shl_nuw_nsw_ctlz_true( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTLZ:%.*]] = sub <2 x i32> , [[TMP0]] +; CHECK-NEXT: ret <2 x i32> [[CTLZ]] +; + %shl = shl nuw nsw <2 x i32> , %0 + %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %shl, i1 true) + ret <2 x i32> %ctlz +} + +define <2 x i32> @vec2_lshr_exact_cttz_true(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_lshr_exact_cttz_true( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTTZ:%.*]] = sub <2 x i32> , [[TMP0]] +; CHECK-NEXT: ret <2 x i32> [[CTTZ]] +; + %lshr = lshr exact <2 x i32> , %0 + %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %lshr, i1 true) + ret <2 x i32> %cttz +} + +define <2 x i32> @vec2_shl_cttz_true(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_shl_cttz_true( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[CTTZ:%.*]] = add <2 x i32> [[TMP0]], +; CHECK-NEXT: ret <2 x i32> [[CTTZ]] +; + %shl = shl <2 x i32> , %0 + %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %shl, i1 true) + ret <2 x i32> %cttz +} + +; negative tests: + +define <2 x i32> @vec2_shl_nsw_ctlz_true_neg(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_shl_nsw_ctlz_true_neg( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[SHL:%.*]] = shl nsw <2 x i32> , [[TMP0]] +; CHECK-NEXT: [[CTLZ:%.*]] = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[SHL]], i1 true), !range [[RNG0:![0-9]+]] +; CHECK-NEXT: ret <2 x i32> [[CTLZ]] +; + %shl = shl nsw <2 x i32> , %0 + %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %shl, i1 true) + ret <2 x i32> %ctlz +} + +define <2 x i32> @vec2_lshr_ctlz_false_neg(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_lshr_ctlz_false_neg( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[DIV:%.*]] = lshr <2 x i32> , [[TMP0]] +; CHECK-NEXT: [[CTLZ:%.*]] = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[DIV]], i1 false), !range [[RNG1:![0-9]+]] +; CHECK-NEXT: ret <2 x i32> [[CTLZ]] +; + %div = lshr <2 x i32> , %0 + %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %div, i1 false) + ret <2 x i32> %ctlz +} + +define <2 x i32> @vec2_shl_ctlz_false_neg(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_shl_ctlz_false_neg( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[SHL:%.*]] = shl <2 x i32> , [[TMP0]] +; CHECK-NEXT: [[CTLZ:%.*]] = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> [[SHL]], i1 false), !range [[RNG2:![0-9]+]] +; CHECK-NEXT: ret <2 x i32> [[CTLZ]] +; + %shl = shl <2 x i32> , %0 + %ctlz = call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %shl, i1 false) + ret <2 x i32> %ctlz +} + +define <2 x i32> @vec2_lshr_cttz_false_neg(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_lshr_cttz_false_neg( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[LSHR:%.*]] = lshr <2 x i32> , [[TMP0]] +; CHECK-NEXT: [[CTTZ:%.*]] = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[LSHR]], i1 false), !range [[RNG2]] +; CHECK-NEXT: ret <2 x i32> [[CTTZ]] +; + %lshr = lshr <2 x i32> , %0 + %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %lshr, i1 false) + ret <2 x i32> %cttz +} + +define <2 x i32> @vec2_shl_cttz_false_neg(<2 x i32>) { +; CHECK-LABEL: define <2 x i32> @vec2_shl_cttz_false_neg( +; CHECK-SAME: <2 x i32> [[TMP0:%.*]]) { +; CHECK-NEXT: [[SHL:%.*]] = shl <2 x i32> , [[TMP0]] +; CHECK-NEXT: [[CTTZ:%.*]] = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> [[SHL]], i1 false), !range [[RNG3:![0-9]+]] +; CHECK-NEXT: ret <2 x i32> [[CTTZ]] +; + %shl = shl <2 x i32> , %0 + %cttz = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %shl, i1 false) + ret <2 x i32> %cttz +} + +define i32 @lshr_ctlz_faslse_neg(i32) { +; CHECK-LABEL: define i32 @lshr_ctlz_faslse_neg( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[LSHR:%.*]] = lshr i32 8387584, [[TMP0]] +; CHECK-NEXT: [[CTLZ:%.*]] = call i32 @llvm.ctlz.i32(i32 [[LSHR]], i1 false), !range [[RNG1]] +; CHECK-NEXT: ret i32 [[CTLZ]] +; + %lshr = lshr i32 8387584, %0 + %ctlz = call i32 @llvm.ctlz.i32(i32 %lshr, i1 false) + ret i32 %ctlz +} + +define i32 @shl_ctlz_false_neg(i32) { +; CHECK-LABEL: define i32 @shl_ctlz_false_neg( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[SHL:%.*]] = shl i32 8387584, [[TMP0]] +; CHECK-NEXT: [[CTLZ:%.*]] = call i32 @llvm.ctlz.i32(i32 [[SHL]], i1 false), !range [[RNG2]] +; CHECK-NEXT: ret i32 [[CTLZ]] +; + %shl = shl i32 8387584, %0 + %ctlz = call i32 @llvm.ctlz.i32(i32 %shl, i1 false) + ret i32 %ctlz +} + +define i32 @lshr_cttz_false_neg(i32) { +; CHECK-LABEL: define i32 @lshr_cttz_false_neg( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[LSHR:%.*]] = lshr i32 8387584, [[TMP0]] +; CHECK-NEXT: [[CTTZ:%.*]] = call i32 @llvm.cttz.i32(i32 [[LSHR]], i1 false), !range [[RNG2]] +; CHECK-NEXT: ret i32 [[CTTZ]] +; + %lshr = lshr i32 8387584, %0 + %cttz = call i32 @llvm.cttz.i32(i32 %lshr, i1 false) + ret i32 %cttz +} + +define i32 @shl_cttz_false_neg(i32) { +; CHECK-LABEL: define i32 @shl_cttz_false_neg( +; CHECK-SAME: i32 [[TMP0:%.*]]) { +; CHECK-NEXT: [[SHL:%.*]] = shl i32 8387584, [[TMP0]] +; CHECK-NEXT: [[CTTZ:%.*]] = call i32 @llvm.cttz.i32(i32 [[SHL]], i1 false), !range [[RNG4:![0-9]+]] +; CHECK-NEXT: ret i32 [[CTTZ]] +; + %shl = shl i32 8387584, %0 + %cttz = call i32 @llvm.cttz.i32(i32 %shl, i1 false) + ret i32 %cttz +} +;. +; CHECK: [[RNG0]] = !{i32 1, i32 33} +; CHECK: [[RNG1]] = !{i32 9, i32 33} +; CHECK: [[RNG2]] = !{i32 0, i32 33} +; CHECK: [[RNG3]] = !{i32 3, i32 33} +; CHECK: [[RNG4]] = !{i32 10, i32 33} +;. diff --git a/llvm/test/Transforms/InstCombine/icmp-gep.ll b/llvm/test/Transforms/InstCombine/icmp-gep.ll index 7d266b7b246a..99c784d15eb3 100644 --- a/llvm/test/Transforms/InstCombine/icmp-gep.ll +++ b/llvm/test/Transforms/InstCombine/icmp-gep.ll @@ -397,6 +397,52 @@ define i1 @test61_as1(ptr addrspace(1) %foo, i16 %i, i16 %j) { ; Don't transform non-inbounds GEPs. } +define i1 @test60_extra_use(ptr %foo, i64 %i, i64 %j) { +; CHECK-LABEL: @test60_extra_use( +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[FOO:%.*]], i64 [[I:%.*]] +; CHECK-NEXT: [[GEP2:%.*]] = getelementptr inbounds i16, ptr [[FOO]], i64 [[J:%.*]] +; CHECK-NEXT: call void @use(ptr [[GEP1]]) +; CHECK-NEXT: call void @use(ptr [[GEP2]]) +; CHECK-NEXT: [[CMP:%.*]] = icmp ult ptr [[GEP1]], [[GEP2]] +; CHECK-NEXT: ret i1 [[CMP]] +; + %gep1 = getelementptr inbounds i32, ptr %foo, i64 %i + %gep2 = getelementptr inbounds i16, ptr %foo, i64 %j + call void @use(ptr %gep1) + call void @use(ptr %gep2) + %cmp = icmp ult ptr %gep1, %gep2 + ret i1 %cmp +} + +define i1 @test60_extra_use_const_operands_inbounds(ptr %foo, i64 %i, i64 %j) { +; CHECK-LABEL: @test60_extra_use_const_operands_inbounds( +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr inbounds i32, ptr [[FOO:%.*]], i64 1 +; CHECK-NEXT: call void @use(ptr nonnull [[GEP1]]) +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[J:%.*]], 2 +; CHECK-NEXT: ret i1 [[CMP]] +; + %gep1 = getelementptr inbounds i32, ptr %foo, i64 1 + %gep2 = getelementptr inbounds i16, ptr %foo, i64 %j + call void @use(ptr %gep1) + %cmp = icmp eq ptr %gep1, %gep2 + ret i1 %cmp +} + +define i1 @test60_extra_use_const_operands_no_inbounds(ptr %foo, i64 %i, i64 %j) { +; CHECK-LABEL: @test60_extra_use_const_operands_no_inbounds( +; CHECK-NEXT: [[GEP1:%.*]] = getelementptr i32, ptr [[FOO:%.*]], i64 1 +; CHECK-NEXT: call void @use(ptr [[GEP1]]) +; CHECK-NEXT: [[GEP2_IDX_MASK:%.*]] = and i64 [[J:%.*]], 9223372036854775807 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[GEP2_IDX_MASK]], 2 +; CHECK-NEXT: ret i1 [[CMP]] +; + %gep1 = getelementptr i32, ptr %foo, i64 1 + %gep2 = getelementptr i16, ptr %foo, i64 %j + call void @use(ptr %gep1) + %cmp = icmp eq ptr %gep1, %gep2 + ret i1 %cmp +} + define i1 @test_scalable_same(ptr %x) { ; CHECK-LABEL: @test_scalable_same( ; CHECK-NEXT: ret i1 false diff --git a/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll index 574f3e23a4ec..7f4603881f23 100644 --- a/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll +++ b/llvm/test/Transforms/InstCombine/icmp-uge-of-not-of-shl-allones-by-bits-and-val-to-icmp-eq-of-lshr-val-by-bits-and-0.ll @@ -101,7 +101,7 @@ define i1 @both(i8 %bits0, i8 %bits1) { ; CHECK-LABEL: @both( ; CHECK-NEXT: [[T0:%.*]] = shl nsw i8 -1, [[BITS0:%.*]] ; CHECK-NEXT: [[T2:%.*]] = shl nsw i8 -1, [[BITS1:%.*]] -; CHECK-NEXT: [[R:%.*]] = icmp uge i8 [[T2]], [[T0]] +; CHECK-NEXT: [[R:%.*]] = icmp ule i8 [[T0]], [[T2]] ; CHECK-NEXT: ret i1 [[R]] ; %t0 = shl i8 -1, %bits0 diff --git a/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll b/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll index 5f0af994f0eb..c7a45c5cdc11 100644 --- a/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll +++ b/llvm/test/Transforms/InstCombine/icmp-ult-of-not-of-shl-allones-by-bits-and-val-to-icmp-ne-of-lshr-val-by-bits-and-0.ll @@ -101,7 +101,7 @@ define i1 @both(i8 %bits0, i8 %bits1) { ; CHECK-LABEL: @both( ; CHECK-NEXT: [[T0:%.*]] = shl nsw i8 -1, [[BITS0:%.*]] ; CHECK-NEXT: [[T2:%.*]] = shl nsw i8 -1, [[BITS1:%.*]] -; CHECK-NEXT: [[R:%.*]] = icmp ult i8 [[T2]], [[T0]] +; CHECK-NEXT: [[R:%.*]] = icmp ugt i8 [[T0]], [[T2]] ; CHECK-NEXT: ret i1 [[R]] ; %t0 = shl i8 -1, %bits0 diff --git a/llvm/test/Transforms/InstCombine/known-bits.ll b/llvm/test/Transforms/InstCombine/known-bits.ll index cf560d7f333f..e346330aa5b1 100644 --- a/llvm/test/Transforms/InstCombine/known-bits.ll +++ b/llvm/test/Transforms/InstCombine/known-bits.ll @@ -48,4 +48,246 @@ define void @test_udiv(i8 %x) { ret void } +define i8 @test_cond(i8 %x) { +; CHECK-LABEL: @test_cond( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: br i1 [[CMP]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: ret i8 -4 +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp eq i8 %and, 0 + br i1 %cmp, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_inv(i8 %x) { +; CHECK-LABEL: @test_cond_inv( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[AND]], 0 +; CHECK-NEXT: call void @use(i1 [[CMP]]) +; CHECK-NEXT: br i1 [[CMP]], label [[EXIT:%.*]], label [[IF:%.*]] +; CHECK: if: +; CHECK-NEXT: ret i8 -4 +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp ne i8 %and, 0 + call void @use(i1 %cmp) + br i1 %cmp, label %exit, label %if + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_and(i8 %x, i1 %c) { +; CHECK-LABEL: @test_cond_and( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[COND:%.*]] = and i1 [[CMP]], [[C:%.*]] +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp eq i8 %and, 0 + %cond = and i1 %cmp, %c + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_and_commuted(i8 %x, i1 %c1, i1 %c2) { +; CHECK-LABEL: @test_cond_and_commuted( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[C3:%.*]] = and i1 [[C1:%.*]], [[C2:%.*]] +; CHECK-NEXT: [[COND:%.*]] = and i1 [[C3]], [[CMP]] +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp eq i8 %and, 0 + %c3 = and i1 %c1, %c2 + %cond = and i1 %c3, %cmp + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_logical_and(i8 %x, i1 %c) { +; CHECK-LABEL: @test_cond_logical_and( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP]], i1 [[C:%.*]], i1 false +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp eq i8 %and, 0 + %cond = select i1 %cmp, i1 %c, i1 false + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_or_invalid(i8 %x, i1 %c) { +; CHECK-LABEL: @test_cond_or_invalid( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[COND:%.*]] = or i1 [[CMP]], [[C:%.*]] +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp eq i8 %and, 0 + %cond = or i1 %cmp, %c + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_inv_or(i8 %x, i1 %c) { +; CHECK-LABEL: @test_cond_inv_or( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[AND]], 0 +; CHECK-NEXT: [[COND:%.*]] = or i1 [[CMP]], [[C:%.*]] +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp ne i8 %and, 0 + %cond = or i1 %cmp, %c + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_inv_logical_or(i8 %x, i1 %c) { +; CHECK-LABEL: @test_cond_inv_logical_or( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP_NOT:%.*]] = icmp eq i8 [[AND]], 0 +; CHECK-NEXT: [[COND:%.*]] = select i1 [[CMP_NOT]], i1 [[C:%.*]], i1 false +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp ne i8 %and, 0 + %cond = select i1 %cmp, i1 false, i1 %c + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +define i8 @test_cond_inv_and_invalid(i8 %x, i1 %c) { +; CHECK-LABEL: @test_cond_inv_and_invalid( +; CHECK-NEXT: [[AND:%.*]] = and i8 [[X:%.*]], 3 +; CHECK-NEXT: [[CMP:%.*]] = icmp ne i8 [[AND]], 0 +; CHECK-NEXT: [[COND:%.*]] = and i1 [[CMP]], [[C:%.*]] +; CHECK-NEXT: br i1 [[COND]], label [[IF:%.*]], label [[EXIT:%.*]] +; CHECK: if: +; CHECK-NEXT: [[OR1:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR1]] +; CHECK: exit: +; CHECK-NEXT: [[OR2:%.*]] = or i8 [[X]], -4 +; CHECK-NEXT: ret i8 [[OR2]] +; + %and = and i8 %x, 3 + %cmp = icmp ne i8 %and, 0 + %cond = and i1 %cmp, %c + br i1 %cond, label %if, label %exit + +if: + %or1 = or i8 %x, -4 + ret i8 %or1 + +exit: + %or2 = or i8 %x, -4 + ret i8 %or2 +} + +declare void @use(i1) declare void @sink(i8) diff --git a/llvm/test/Transforms/InstCombine/not.ll b/llvm/test/Transforms/InstCombine/not.ll index 847625af8b9c..3b0e5b4412fb 100644 --- a/llvm/test/Transforms/InstCombine/not.ll +++ b/llvm/test/Transforms/InstCombine/not.ll @@ -39,7 +39,7 @@ define i1 @invert_fcmp(float %X, float %Y) { define i1 @not_not_cmp(i32 %a, i32 %b) { ; CHECK-LABEL: @not_not_cmp( -; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[B:%.*]], [[A:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: ret i1 [[CMP]] ; %nota = xor i32 %a, -1 @@ -50,7 +50,7 @@ define i1 @not_not_cmp(i32 %a, i32 %b) { define <2 x i1> @not_not_cmp_vector(<2 x i32> %a, <2 x i32> %b) { ; CHECK-LABEL: @not_not_cmp_vector( -; CHECK-NEXT: [[CMP:%.*]] = icmp ugt <2 x i32> [[B:%.*]], [[A:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp ult <2 x i32> [[A:%.*]], [[B:%.*]] ; CHECK-NEXT: ret <2 x i1> [[CMP]] ; %nota = xor <2 x i32> %a, @@ -727,9 +727,8 @@ define i8 @bitcast_to_scalar_sext_bool_use2(<4 x i1> %b) { define i1 @invert_both_cmp_operands_add(i32 %a, i32 %b) { ; CHECK-LABEL: @invert_both_cmp_operands_add( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[NOT_A:%.*]] = xor i32 [[A:%.*]], -1 -; CHECK-NEXT: [[ADD:%.*]] = add i32 [[NOT_A]], [[B:%.*]] -; CHECK-NEXT: [[CMP:%.*]] = icmp sgt i32 [[ADD]], 0 +; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp slt i32 [[TMP0]], -1 ; CHECK-NEXT: ret i1 [[CMP]] ; entry: @@ -742,9 +741,8 @@ entry: define i1 @invert_both_cmp_operands_sub(i32 %a, i32 %b) { ; CHECK-LABEL: @invert_both_cmp_operands_sub( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[NOT_A:%.*]] = xor i32 [[A:%.*]], -1 -; CHECK-NEXT: [[ADD:%.*]] = sub i32 [[NOT_A]], [[B:%.*]] -; CHECK-NEXT: [[CMP:%.*]] = icmp ult i32 [[ADD]], 42 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[A:%.*]], [[B:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp ugt i32 [[TMP0]], -43 ; CHECK-NEXT: ret i1 [[CMP]] ; entry: @@ -757,12 +755,9 @@ entry: define i1 @invert_both_cmp_operands_complex(i1 %x, i32 %a, i32 %b, i32 %c) { ; CHECK-LABEL: @invert_both_cmp_operands_complex( ; CHECK-NEXT: entry: -; CHECK-NEXT: [[NOT_A:%.*]] = xor i32 [[A:%.*]], -1 -; CHECK-NEXT: [[NOT_B:%.*]] = xor i32 [[B:%.*]], -1 -; CHECK-NEXT: [[NOT_C:%.*]] = xor i32 [[C:%.*]], -1 -; CHECK-NEXT: [[ADD:%.*]] = add i32 [[NOT_A]], [[C]] -; CHECK-NEXT: [[SELECT:%.*]] = select i1 [[X:%.*]], i32 [[ADD]], i32 [[NOT_B]] -; CHECK-NEXT: [[CMP:%.*]] = icmp sle i32 [[SELECT]], [[NOT_C]] +; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[A:%.*]], [[C:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[X:%.*]], i32 [[TMP0]], i32 [[B:%.*]] +; CHECK-NEXT: [[CMP:%.*]] = icmp sge i32 [[TMP1]], [[C]] ; CHECK-NEXT: ret i1 [[CMP]] ; entry: diff --git a/llvm/test/Transforms/InstCombine/or.ll b/llvm/test/Transforms/InstCombine/or.ll index fd415682b67f..8c8aab2bcba6 100644 --- a/llvm/test/Transforms/InstCombine/or.ll +++ b/llvm/test/Transforms/InstCombine/or.ll @@ -1608,3 +1608,16 @@ define i8 @drop_disjoint(i8 %x) { %b = or disjoint i8 %a, 1 ret i8 %b } + +; Make sure we drop disjoint when combining the Ors. +define i32 @assoc_cast_assoc_disjoint(i16 %x) { +; CHECK-LABEL: @assoc_cast_assoc_disjoint( +; CHECK-NEXT: [[B:%.*]] = zext i16 [[X:%.*]] to i32 +; CHECK-NEXT: [[C:%.*]] = or i32 [[B]], 65537 +; CHECK-NEXT: ret i32 [[C]] +; + %a = or i16 %x, 1 + %b = zext i16 %a to i32 + %c = or disjoint i32 %b, 65536 + ret i32 %c +} diff --git a/llvm/test/Transforms/InstSimplify/returned.ll b/llvm/test/Transforms/InstSimplify/returned.ll index 94a98ac6cb05..2da1052e7d4d 100644 --- a/llvm/test/Transforms/InstSimplify/returned.ll +++ b/llvm/test/Transforms/InstSimplify/returned.ll @@ -25,6 +25,31 @@ define i1 @gep3() { ret i1 %equal } +define <8 x i1> @returned_vec_arg_casted(<2 x i32> %a) { +; CHECK-LABEL: @returned_vec_arg_casted( +; CHECK-NEXT: [[X:%.*]] = call <8 x i8> @passthru_8i8v_from_2i32v(<2 x i32> [[A:%.*]]) +; CHECK-NEXT: [[C:%.*]] = icmp slt <8 x i8> [[X]], zeroinitializer +; CHECK-NEXT: ret <8 x i1> [[C]] +; + %x = call <8 x i8> @passthru_8i8v_from_2i32v(<2 x i32> %a) + %C = icmp slt <8 x i8> %x, zeroinitializer + ret <8 x i1> %C +} + +define <8 x i1> @returned_vec_arg_casted2(<2 x i32> %a) { +; CHECK-LABEL: @returned_vec_arg_casted2( +; CHECK-NEXT: [[OR:%.*]] = or <2 x i32> [[A:%.*]], +; CHECK-NEXT: [[X:%.*]] = call <8 x i8> @passthru_8i8v_from_2i32v(<2 x i32> [[OR]]) +; CHECK-NEXT: [[C:%.*]] = icmp ne <8 x i8> [[X]], zeroinitializer +; CHECK-NEXT: ret <8 x i1> [[C]] +; + %or = or <2 x i32> %a, + %x = call <8 x i8> @passthru_8i8v_from_2i32v(<2 x i32> %or) + %C = icmp ne <8 x i8> %x, zeroinitializer + ret <8 x i1> %C +} + +declare <8 x i8> @passthru_8i8v_from_2i32v(<2 x i32> returned) declare ptr @func1(ptr returned) nounwind readnone willreturn declare ptr @func2(ptr returned) nounwind readnone willreturn diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll index dd08172be1b8..22df002dd62c 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles-inseltpoison.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll index 2e8a7cf42ac5..399fa5298b7c 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/binopshuffles.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll index ab70d623470c..224a0693bf21 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/fixed-deinterleave-intrinsics.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 ; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=NEON ; RUN: opt < %s -interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible-sve -S | FileCheck %s --check-prefix=SVE-FIXED +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=NEON +; RUN: opt < %s -passes=interleaved-access -mtriple=aarch64-linux-gnu -mattr=+sve -force-streaming-compatible-sve -S | FileCheck %s --check-prefix=SVE-FIXED target triple = "aarch64-linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll index db031ed12b7e..e48dc5d3051b 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user-inseltpoison.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll index af472630b951..ea33590cb241 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-extract-user.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll index 77fbb8e7f2ca..14986d9eb85c 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses-inseltpoison.ll @@ -1,5 +1,7 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefix=NEON ; RUN: opt < %s -mattr=-neon -interleaved-access -S | FileCheck %s -check-prefix=NO_NEON +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s -check-prefix=NEON +; RUN: opt < %s -mattr=-neon -passes=interleaved-access -S | FileCheck %s -check-prefix=NO_NEON target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll index 77fbb8e7f2ca..14986d9eb85c 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/interleaved-accesses.ll @@ -1,5 +1,7 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s -check-prefix=NEON ; RUN: opt < %s -mattr=-neon -interleaved-access -S | FileCheck %s -check-prefix=NO_NEON +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s -check-prefix=NEON +; RUN: opt < %s -mattr=-neon -passes=interleaved-access -S | FileCheck %s -check-prefix=NO_NEON target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" target triple = "aarch64--linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll index c04464b2ca9d..6353bf10d57c 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/scalable-deinterleave-intrinsics.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2 ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target triple = "aarch64-linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll index 94f63e5921f8..feb22aa1a376 100644 --- a/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll +++ b/llvm/test/Transforms/InterleavedAccess/AArch64/sve-interleaved-accesses.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -interleaved-access -S | FileCheck %s +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s target triple = "aarch64-linux-gnu" diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user-inseltpoison.ll index 3ea00de0dcb1..77b376b3580c 100644 --- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user-inseltpoison.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64" target triple = "arm---eabi" diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll index 0c56f71cc369..893494167872 100644 --- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll +++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-extract-user.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64" target triple = "arm---eabi" diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll index 9ea1c5e94a98..aed843723189 100644 --- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses-inseltpoison.ll @@ -2,6 +2,9 @@ ; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON ; RUN: opt < %s -mattr=+mve.fp -interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE ; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE +; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON +; RUN: opt < %s -mattr=+mve.fp -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64" target triple = "arm---eabi" diff --git a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll index c84d759ae0fb..8123ea5bbe9f 100644 --- a/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll +++ b/llvm/test/Transforms/InterleavedAccess/ARM/interleaved-accesses.ll @@ -2,6 +2,9 @@ ; RUN: opt < %s -mattr=+neon -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON ; RUN: opt < %s -mattr=+mve.fp -interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE ; RUN: opt < %s -interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE +; RUN: opt < %s -mattr=+neon -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NEON +; RUN: opt < %s -mattr=+mve.fp -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-MVE +; RUN: opt < %s -passes=interleaved-access -S | FileCheck %s --check-prefix=CHECK-NONE target datalayout = "e-m:e-p:32:32-i64:64-v128:64:128-n32-S64" target triple = "arm---eabi" diff --git a/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll index 736a7ce4ecf0..ea2d1ca722ca 100644 --- a/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll +++ b/llvm/test/Transforms/InterleavedAccess/RISCV/interleaved-accesses.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=riscv32 -mattr=+v -interleaved-access -S | FileCheck %s --check-prefix=RV32 ; RUN: opt < %s -mtriple=riscv64 -mattr=+v -interleaved-access -S | FileCheck %s --check-prefix=RV64 +; RUN: opt < %s -mtriple=riscv32 -mattr=+v -passes=interleaved-access -S | FileCheck %s --check-prefix=RV32 +; RUN: opt < %s -mtriple=riscv64 -mattr=+v -passes=interleaved-access -S | FileCheck %s --check-prefix=RV64 define void @load_factor2(ptr %ptr) { ; RV32-LABEL: @load_factor2( diff --git a/llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll b/llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll index e7b7ec052f45..ac3cab6638b8 100644 --- a/llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll +++ b/llvm/test/Transforms/InterleavedAccess/RISCV/zve32x.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=riscv64 -mattr=+zve32x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVE32X ; RUN: opt < %s -mtriple=riscv64 -mattr=+zve64x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVE64X +; RUN: opt < %s -mtriple=riscv64 -mattr=+zve32x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVE32X +; RUN: opt < %s -mtriple=riscv64 -mattr=+zve64x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVE64X define <4 x i1> @load_large_vector(ptr %p) { ; ZVE32X-LABEL: @load_large_vector( diff --git a/llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll b/llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll index 9c896796760f..a94e6a70e79e 100644 --- a/llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll +++ b/llvm/test/Transforms/InterleavedAccess/RISCV/zvl32b.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl32b -interleaved-access -S | FileCheck %s -check-prefix=ZVL32B ; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl128b -interleaved-access -S | FileCheck %s -check-prefix=ZVL128B +; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl32b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVL32B +; RUN: opt < %s -mtriple=riscv32 -mattr=+zve32x,+zvl128b -passes=interleaved-access -S | FileCheck %s -check-prefix=ZVL128B ; Make sure that we don't lower interleaved loads that won't fit into the minimum vlen diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleave-load-extract-shuffle-changes.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleave-load-extract-shuffle-changes.ll index a4b4e3748b46..167d282edb3e 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleave-load-extract-shuffle-changes.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleave-load-extract-shuffle-changes.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt -interleaved-access -S %s | FileCheck %s +; RUN: opt -passes=interleaved-access -S %s | FileCheck %s target datalayout = "e-m:o-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-apple-macosx10.15.0" diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll index b4128dda7cee..13076c7ffec9 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx-inseltpoison.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -passes=interleaved-access -S | FileCheck %s ; This file tests the function `llvm::lowerInterleavedLoad/Store`. diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll index ec905a999dca..6972afe486bd 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleaved-accesses-64bits-avx.ll @@ -1,5 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx -passes=interleaved-access -S | FileCheck %s ; This file tests the function `llvm::lowerInterleavedLoad/Store`. diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad-inseltpoison.ll index 9b38f2dfbabd..327a1d2c6a66 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad-inseltpoison.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -passes=interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -passes=interleaved-access -S | FileCheck %s define <32 x i8> @interleaved_load_vf32_i8_stride3(ptr %ptr){ ; CHECK-LABEL: @interleaved_load_vf32_i8_stride3( diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll index 86c186276258..68d8b9475517 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedLoad.ll @@ -1,6 +1,8 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -passes=interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx512f -mattr=+avx512bw -mattr=+avx512vl -passes=interleaved-access -S | FileCheck %s define <32 x i8> @interleaved_load_vf32_i8_stride3(ptr %ptr){ ; CHECK-LABEL: @interleaved_load_vf32_i8_stride3( diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll index 301abda5708c..d1eea7a0d1bf 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore-inseltpoison.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -passes=interleaved-access -S | FileCheck %s define void @interleaved_store_vf32_i8_stride4(<32 x i8> %x1, <32 x i8> %x2, <32 x i8> %x3, <32 x i8> %x4, ptr %p) { ; CHECK-LABEL: @interleaved_store_vf32_i8_stride4( diff --git a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll index 75595463c740..3e7bc4130a8c 100644 --- a/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll +++ b/llvm/test/Transforms/InterleavedAccess/X86/interleavedStore.ll @@ -1,6 +1,7 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py ; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -interleaved-access -S | FileCheck %s +; RUN: opt < %s -mtriple=x86_64-pc-linux -mattr=+avx2 -passes=interleaved-access -S | FileCheck %s define void @interleaved_store_vf32_i8_stride4(<32 x i8> %x1, <32 x i8> %x2, <32 x i8> %x3, <32 x i8> %x4, ptr %p) { ; CHECK-LABEL: @interleaved_store_vf32_i8_stride4( diff --git a/llvm/test/Transforms/LICM/pr64897.ll b/llvm/test/Transforms/LICM/pr64897.ll index 1ce78f15d797..12b11eb6912b 100644 --- a/llvm/test/Transforms/LICM/pr64897.ll +++ b/llvm/test/Transforms/LICM/pr64897.ll @@ -1,7 +1,6 @@ ; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 ; RUN: opt -S -passes=licm < %s | FileCheck %s -; FIXME: This is a miscompile. define void @test(i1 %c, i8 %x) { ; CHECK-LABEL: define void @test( ; CHECK-SAME: i1 [[C:%.*]], i8 [[X:%.*]]) { @@ -10,17 +9,16 @@ define void @test(i1 %c, i8 %x) { ; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8 ; CHECK-NEXT: [[P_COPY:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 8 ; CHECK-NEXT: [[P2:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 12 -; CHECK-NEXT: [[P2_PROMOTED:%.*]] = load i8, ptr [[P2]], align 1 ; CHECK-NEXT: br label [[LOOP:%.*]] ; CHECK: loop: -; CHECK-NEXT: [[TMP0:%.*]] = phi i8 [ 0, [[LOOP]] ], [ [[P2_PROMOTED]], [[START:%.*]] ] ; CHECK-NEXT: store i32 286331153, ptr [[P]], align 4 ; CHECK-NEXT: store i32 34, ptr [[P_COPY]], align 4 ; CHECK-NEXT: store i64 3689348814741910323, ptr [[P_COPY]], align 4 -; CHECK-NEXT: call void @use(i8 [[TMP0]]) +; CHECK-NEXT: [[VAL:%.*]] = load i8, ptr [[P2]], align 1 +; CHECK-NEXT: call void @use(i8 [[VAL]]) +; CHECK-NEXT: store i8 0, ptr [[P2]], align 1 ; CHECK-NEXT: br i1 [[C]], label [[LOOP]], label [[EXIT:%.*]] ; CHECK: exit: -; CHECK-NEXT: store i8 0, ptr [[P2]], align 1 ; CHECK-NEXT: ret void ; start: diff --git a/llvm/test/Transforms/LoopPredication/basic.ll b/llvm/test/Transforms/LoopPredication/basic.ll index 2a99963ad078..27c8bc99c407 100644 --- a/llvm/test/Transforms/LoopPredication/basic.ll +++ b/llvm/test/Transforms/LoopPredication/basic.ll @@ -1681,7 +1681,7 @@ define i32 @ne_latch_zext(ptr %array, i32 %length, i16 %n16) { ; CHECK-LABEL: @ne_latch_zext( ; CHECK-NEXT: loop.preheader: ; CHECK-NEXT: [[N:%.*]] = zext i16 [[N16:%.*]] to i32 -; CHECK-NEXT: [[NPLUS1:%.*]] = add i32 [[N]], 1 +; CHECK-NEXT: [[NPLUS1:%.*]] = add nuw nsw i32 [[N]], 1 ; CHECK-NEXT: [[TMP0:%.*]] = icmp ule i32 [[NPLUS1]], [[LENGTH:%.*]] ; CHECK-NEXT: [[TMP1:%.*]] = icmp ult i32 0, [[LENGTH]] ; CHECK-NEXT: [[TMP2:%.*]] = and i1 [[TMP1]], [[TMP0]] diff --git a/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll b/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll index c9b66896eac7..0203abe69ac2 100644 --- a/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll +++ b/llvm/test/Transforms/LoopStrengthReduce/lsr-term-fold.ll @@ -453,7 +453,7 @@ define void @non_branch_terminator(ptr %a) { ; CHECK-NEXT: [[UGLYGEP2]] = getelementptr i8, ptr [[LSR_IV1]], i64 4 ; CHECK-NEXT: [[LSR_IV_NEXT3]] = add nsw i64 [[LSR_IV2]], -1 ; CHECK-NEXT: switch i64 [[LSR_IV2]], label [[FOR_BODY]] [ -; CHECK-NEXT: i64 0, label [[FOR_END:%.*]] +; CHECK-NEXT: i64 0, label [[FOR_END:%.*]] ; CHECK-NEXT: ] ; CHECK: for.end: ; CHECK-NEXT: ret void @@ -473,3 +473,117 @@ for.body: ; preds = %for.body, %entry for.end: ; preds = %for.body ret void } + +define void @expensive_expand_short_tc(ptr %a, i32 %offset, i32 %n) { +; CHECK-LABEL: @expensive_expand_short_tc( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 84 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 +; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 +; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[OFFSET:%.*]] to i64 +; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP2]], [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], 84 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[LSR_IV1:%.*]] = phi ptr [ [[UGLYGEP2:%.*]], [[FOR_BODY]] ], [ [[UGLYGEP]], [[ENTRY:%.*]] ] +; CHECK-NEXT: store i32 1, ptr [[LSR_IV1]], align 4 +; CHECK-NEXT: [[UGLYGEP2]] = getelementptr i8, ptr [[LSR_IV1]], i32 [[OFFSET]] +; CHECK-NEXT: [[LSR_FOLD_TERM_COND_REPLACED_TERM_COND:%.*]] = icmp eq ptr [[UGLYGEP2]], [[SCEVGEP]] +; CHECK-NEXT: br i1 [[LSR_FOLD_TERM_COND_REPLACED_TERM_COND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !prof [[PROF0:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void +; +entry: + %uglygep = getelementptr i8, ptr %a, i64 84 + br label %for.body + +for.body: ; preds = %for.body, %entry + %lsr.iv1 = phi ptr [ %uglygep2, %for.body ], [ %uglygep, %entry ] + %lsr.iv = phi i32 [ %lsr.iv.next, %for.body ], [ 0, %entry ] + store i32 1, ptr %lsr.iv1, align 4 + %lsr.iv.next = add nsw i32 %lsr.iv, 1 + %uglygep2 = getelementptr i8, ptr %lsr.iv1, i32 %offset + %exitcond.not = icmp eq i32 %lsr.iv.next, %n + br i1 %exitcond.not, label %for.end, label %for.body, !prof !{!"branch_weights", i32 1, i32 3} + +for.end: ; preds = %for.body + ret void +} + +define void @expensive_expand_long_tc(ptr %a, i32 %offset, i32 %n) { +; CHECK-LABEL: @expensive_expand_long_tc( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 84 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 +; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 +; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[OFFSET:%.*]] to i64 +; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP2]], [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], 84 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[LSR_IV1:%.*]] = phi ptr [ [[UGLYGEP2:%.*]], [[FOR_BODY]] ], [ [[UGLYGEP]], [[ENTRY:%.*]] ] +; CHECK-NEXT: store i32 1, ptr [[LSR_IV1]], align 4 +; CHECK-NEXT: [[UGLYGEP2]] = getelementptr i8, ptr [[LSR_IV1]], i32 [[OFFSET]] +; CHECK-NEXT: [[LSR_FOLD_TERM_COND_REPLACED_TERM_COND:%.*]] = icmp eq ptr [[UGLYGEP2]], [[SCEVGEP]] +; CHECK-NEXT: br i1 [[LSR_FOLD_TERM_COND_REPLACED_TERM_COND]], label [[FOR_END:%.*]], label [[FOR_BODY]], !prof [[PROF1:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: ret void +; +entry: + %uglygep = getelementptr i8, ptr %a, i64 84 + br label %for.body + +for.body: ; preds = %for.body, %entry + %lsr.iv1 = phi ptr [ %uglygep2, %for.body ], [ %uglygep, %entry ] + %lsr.iv = phi i32 [ %lsr.iv.next, %for.body ], [ 0, %entry ] + store i32 1, ptr %lsr.iv1, align 4 + %lsr.iv.next = add nsw i32 %lsr.iv, 1 + %uglygep2 = getelementptr i8, ptr %lsr.iv1, i32 %offset + %exitcond.not = icmp eq i32 %lsr.iv.next, %n + br i1 %exitcond.not, label %for.end, label %for.body, !prof !{!"branch_weights", i32 1, i32 300} + +for.end: ; preds = %for.body + ret void +} + +define void @expensive_expand_unknown_tc(ptr %a, i32 %offset, i32 %n) { +; CHECK-LABEL: @expensive_expand_unknown_tc( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[UGLYGEP:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 84 +; CHECK-NEXT: [[TMP0:%.*]] = add i32 [[N:%.*]], -1 +; CHECK-NEXT: [[TMP1:%.*]] = zext i32 [[TMP0]] to i64 +; CHECK-NEXT: [[TMP2:%.*]] = add nuw nsw i64 [[TMP1]], 1 +; CHECK-NEXT: [[TMP3:%.*]] = sext i32 [[OFFSET:%.*]] to i64 +; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[TMP2]], [[TMP3]] +; CHECK-NEXT: [[TMP5:%.*]] = add nsw i64 [[TMP4]], 84 +; CHECK-NEXT: [[SCEVGEP:%.*]] = getelementptr i8, ptr [[A]], i64 [[TMP5]] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[LSR_IV1:%.*]] = phi ptr [ [[UGLYGEP2:%.*]], [[FOR_BODY]] ], [ [[UGLYGEP]], [[ENTRY:%.*]] ] +; CHECK-NEXT: store i32 1, ptr [[LSR_IV1]], align 4 +; CHECK-NEXT: [[UGLYGEP2]] = getelementptr i8, ptr [[LSR_IV1]], i32 [[OFFSET]] +; CHECK-NEXT: [[LSR_FOLD_TERM_COND_REPLACED_TERM_COND:%.*]] = icmp eq ptr [[UGLYGEP2]], [[SCEVGEP]] +; CHECK-NEXT: br i1 [[LSR_FOLD_TERM_COND_REPLACED_TERM_COND]], label [[FOR_END:%.*]], label [[FOR_BODY]] +; CHECK: for.end: +; CHECK-NEXT: ret void +; +entry: + %uglygep = getelementptr i8, ptr %a, i64 84 + br label %for.body + +for.body: ; preds = %for.body, %entry + %lsr.iv1 = phi ptr [ %uglygep2, %for.body ], [ %uglygep, %entry ] + %lsr.iv = phi i32 [ %lsr.iv.next, %for.body ], [ 0, %entry ] + store i32 1, ptr %lsr.iv1, align 4 + %lsr.iv.next = add nsw i32 %lsr.iv, 1 + %uglygep2 = getelementptr i8, ptr %lsr.iv1, i32 %offset + %exitcond.not = icmp eq i32 %lsr.iv.next, %n + br i1 %exitcond.not, label %for.end, label %for.body + +for.end: ; preds = %for.body + ret void +} diff --git a/llvm/test/Transforms/LoopUnroll/loop-branch-folding.ll b/llvm/test/Transforms/LoopUnroll/loop-branch-folding.ll new file mode 100644 index 000000000000..f0a6c0a954f6 --- /dev/null +++ b/llvm/test/Transforms/LoopUnroll/loop-branch-folding.ll @@ -0,0 +1,936 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -S -passes=simplifycfg | FileCheck %s --check-prefixes=CHECK-CFG +; RUN: opt < %s -S -passes=simplifycfg,loop-unroll --unroll-max-upperbound=17 | FileCheck %s --check-prefixes=CHECK-UNROLL + +; This test designed to check: +; We can still unroll loop with 'pragma unroll' if loop count(trip count) was destroyed by previous optimization. +; For exmaple, in following test, loop condition "Dim < 16" was 'merged' with "Dim == Dims" in folding branches +; at simplifycfg. But if custumer mark the loop with "#pragma unroll", we can still successfully unroll it under +; unroll-max-upperbound. +; +; __device__ void func(int Idx, int *Arr[], int Dims, int *Out) { +; #pragma unroll +; for (int Dim = 0; Dim < 16; ++Dim) { +; if (Dim == Dims) { +; break; +; } +; int divmod = Arr[Dim][Idx]; +; Idx = divmod + 1; +; +; for (int arg = 0; arg < 4; arg++) { +; Out[arg] += Arr[Dim][arg]; +; bar(); +; } +; } +; } + +define void @func(i32 noundef %Idx, ptr noundef %Arr, i32 noundef %Dims, ptr noundef %Out) { +; CHECK-CFG-LABEL: define void @func( +; CHECK-CFG-SAME: i32 noundef [[IDX:%.*]], ptr noundef [[ARR:%.*]], i32 noundef [[DIMS:%.*]], ptr noundef [[OUT:%.*]]) { +; CHECK-CFG-NEXT: entry: +; CHECK-CFG-NEXT: br label [[FOR_COND:%.*]] +; CHECK-CFG: for.cond: +; CHECK-CFG-NEXT: [[DIM_0:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[INC16:%.*]], [[FOR_COND_CLEANUP6:%.*]] ] +; CHECK-CFG-NEXT: [[IDX_ADDR_0:%.*]] = phi i32 [ [[IDX]], [[ENTRY]] ], [ [[ADD:%.*]], [[FOR_COND_CLEANUP6]] ] +; CHECK-CFG-NEXT: [[CMP:%.*]] = icmp sge i32 [[DIM_0]], 16 +; CHECK-CFG-NEXT: [[CMP1:%.*]] = icmp eq i32 [[DIM_0]], [[DIMS]] +; CHECK-CFG-NEXT: [[OR_COND:%.*]] = or i1 [[CMP]], [[CMP1]] +; CHECK-CFG-NEXT: br i1 [[OR_COND]], label [[CLEANUP:%.*]], label [[IF_END:%.*]] +; CHECK-CFG: if.end: +; CHECK-CFG-NEXT: [[IDXPROM:%.*]] = sext i32 [[DIM_0]] to i64 +; CHECK-CFG-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 [[IDXPROM]] +; CHECK-CFG-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8 +; CHECK-CFG-NEXT: [[IDXPROM2:%.*]] = sext i32 [[IDX_ADDR_0]] to i64 +; CHECK-CFG-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i32, ptr [[TMP0]], i64 [[IDXPROM2]] +; CHECK-CFG-NEXT: [[TMP1:%.*]] = load i32, ptr [[ARRAYIDX3]], align 4 +; CHECK-CFG-NEXT: [[ADD]] = add nsw i32 [[TMP1]], 1 +; CHECK-CFG-NEXT: br label [[FOR_COND4:%.*]] +; CHECK-CFG: for.cond4: +; CHECK-CFG-NEXT: [[ARG_0:%.*]] = phi i32 [ 0, [[IF_END]] ], [ [[INC:%.*]], [[FOR_BODY7:%.*]] ] +; CHECK-CFG-NEXT: [[CMP5:%.*]] = icmp slt i32 [[ARG_0]], 4 +; CHECK-CFG-NEXT: br i1 [[CMP5]], label [[FOR_BODY7]], label [[FOR_COND_CLEANUP6]] +; CHECK-CFG: for.cond.cleanup6: +; CHECK-CFG-NEXT: [[INC16]] = add nsw i32 [[DIM_0]], 1 +; CHECK-CFG-NEXT: br label [[FOR_COND]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-CFG: for.body7: +; CHECK-CFG-NEXT: [[TMP2:%.*]] = load ptr, ptr [[ARRAYIDX]], align 8 +; CHECK-CFG-NEXT: [[IDXPROM10:%.*]] = sext i32 [[ARG_0]] to i64 +; CHECK-CFG-NEXT: [[ARRAYIDX11:%.*]] = getelementptr inbounds i32, ptr [[TMP2]], i64 [[IDXPROM10]] +; CHECK-CFG-NEXT: [[TMP3:%.*]] = load i32, ptr [[ARRAYIDX11]], align 4 +; CHECK-CFG-NEXT: [[ARRAYIDX13:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 [[IDXPROM10]] +; CHECK-CFG-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX13]], align 4 +; CHECK-CFG-NEXT: [[ADD14:%.*]] = add nsw i32 [[TMP4]], [[TMP3]] +; CHECK-CFG-NEXT: store i32 [[ADD14]], ptr [[ARRAYIDX13]], align 4 +; CHECK-CFG-NEXT: call void @_Z3barv() +; CHECK-CFG-NEXT: [[INC]] = add nsw i32 [[ARG_0]], 1 +; CHECK-CFG-NEXT: br label [[FOR_COND4]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-CFG: cleanup: +; CHECK-CFG-NEXT: ret void +; +; CHECK-UNROLL-LABEL: define void @func( +; CHECK-UNROLL-SAME: i32 noundef [[IDX:%.*]], ptr noundef [[ARR:%.*]], i32 noundef [[DIMS:%.*]], ptr noundef [[OUT:%.*]]) { +; CHECK-UNROLL-NEXT: entry: +; CHECK-UNROLL-NEXT: br label [[FOR_COND:%.*]] +; CHECK-UNROLL: for.cond: +; CHECK-UNROLL-NEXT: [[CMP1:%.*]] = icmp eq i32 0, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1]], label [[CLEANUP:%.*]], label [[IF_END:%.*]] +; CHECK-UNROLL: if.end: +; CHECK-UNROLL-NEXT: br label [[FOR_COND4:%.*]] +; CHECK-UNROLL: for.cond4: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7:%.*]] +; CHECK-UNROLL: for.cond.cleanup6: +; CHECK-UNROLL-NEXT: [[CMP1_1:%.*]] = icmp eq i32 1, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_1]], label [[CLEANUP]], label [[IF_END_1:%.*]] +; CHECK-UNROLL: if.end.1: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 1 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_1:%.*]] +; CHECK-UNROLL: for.cond4.1: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_12:%.*]] +; CHECK-UNROLL: for.body7.12: +; CHECK-UNROLL-NEXT: [[TMP0:%.*]] = load ptr, ptr [[ARRAYIDX_1]], align 8 +; CHECK-UNROLL-NEXT: [[TMP1:%.*]] = load i32, ptr [[TMP0]], align 4 +; CHECK-UNROLL-NEXT: [[TMP2:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_11:%.*]] = add nsw i32 [[TMP2]], [[TMP1]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_11]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_1:%.*]] +; CHECK-UNROLL: for.body7.1.1: +; CHECK-UNROLL-NEXT: [[TMP3:%.*]] = load ptr, ptr [[ARRAYIDX_1]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_1:%.*]] = getelementptr inbounds i32, ptr [[TMP3]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP4:%.*]] = load i32, ptr [[ARRAYIDX11_1_1]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP5:%.*]] = load i32, ptr [[ARRAYIDX13_1_1]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_1:%.*]] = add nsw i32 [[TMP5]], [[TMP4]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_1]], ptr [[ARRAYIDX13_1_1]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_1:%.*]] +; CHECK-UNROLL: for.body7.2.1: +; CHECK-UNROLL-NEXT: [[TMP6:%.*]] = load ptr, ptr [[ARRAYIDX_1]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_1:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP7:%.*]] = load i32, ptr [[ARRAYIDX11_2_1]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP8:%.*]] = load i32, ptr [[ARRAYIDX13_2_1]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_1:%.*]] = add nsw i32 [[TMP8]], [[TMP7]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_1]], ptr [[ARRAYIDX13_2_1]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_1:%.*]] +; CHECK-UNROLL: for.body7.3.1: +; CHECK-UNROLL-NEXT: [[TMP9:%.*]] = load ptr, ptr [[ARRAYIDX_1]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_1:%.*]] = getelementptr inbounds i32, ptr [[TMP9]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP10:%.*]] = load i32, ptr [[ARRAYIDX11_3_1]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP11:%.*]] = load i32, ptr [[ARRAYIDX13_3_1]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_1:%.*]] = add nsw i32 [[TMP11]], [[TMP10]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_1]], ptr [[ARRAYIDX13_3_1]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4:%.*]], label [[FOR_COND_CLEANUP6_1:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.1: +; CHECK-UNROLL-NEXT: [[CMP1_2:%.*]] = icmp eq i32 2, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_2]], label [[CLEANUP]], label [[IF_END_2:%.*]] +; CHECK-UNROLL: if.end.2: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 2 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_2:%.*]] +; CHECK-UNROLL: for.cond4.2: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_24:%.*]] +; CHECK-UNROLL: for.body7.24: +; CHECK-UNROLL-NEXT: [[TMP12:%.*]] = load ptr, ptr [[ARRAYIDX_2]], align 8 +; CHECK-UNROLL-NEXT: [[TMP13:%.*]] = load i32, ptr [[TMP12]], align 4 +; CHECK-UNROLL-NEXT: [[TMP14:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_23:%.*]] = add nsw i32 [[TMP14]], [[TMP13]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_23]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_2:%.*]] +; CHECK-UNROLL: for.body7.1.2: +; CHECK-UNROLL-NEXT: [[TMP15:%.*]] = load ptr, ptr [[ARRAYIDX_2]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_2:%.*]] = getelementptr inbounds i32, ptr [[TMP15]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP16:%.*]] = load i32, ptr [[ARRAYIDX11_1_2]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP17:%.*]] = load i32, ptr [[ARRAYIDX13_1_2]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_2:%.*]] = add nsw i32 [[TMP17]], [[TMP16]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_2]], ptr [[ARRAYIDX13_1_2]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_2:%.*]] +; CHECK-UNROLL: for.body7.2.2: +; CHECK-UNROLL-NEXT: [[TMP18:%.*]] = load ptr, ptr [[ARRAYIDX_2]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_2:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP19:%.*]] = load i32, ptr [[ARRAYIDX11_2_2]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP20:%.*]] = load i32, ptr [[ARRAYIDX13_2_2]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_2:%.*]] = add nsw i32 [[TMP20]], [[TMP19]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_2]], ptr [[ARRAYIDX13_2_2]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_2:%.*]] +; CHECK-UNROLL: for.body7.3.2: +; CHECK-UNROLL-NEXT: [[TMP21:%.*]] = load ptr, ptr [[ARRAYIDX_2]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_2:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP22:%.*]] = load i32, ptr [[ARRAYIDX11_3_2]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP23:%.*]] = load i32, ptr [[ARRAYIDX13_3_2]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_2:%.*]] = add nsw i32 [[TMP23]], [[TMP22]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_2]], ptr [[ARRAYIDX13_3_2]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_2:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.2: +; CHECK-UNROLL-NEXT: [[CMP1_3:%.*]] = icmp eq i32 3, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_3]], label [[CLEANUP]], label [[IF_END_3:%.*]] +; CHECK-UNROLL: if.end.3: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 3 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_3:%.*]] +; CHECK-UNROLL: for.cond4.3: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_36:%.*]] +; CHECK-UNROLL: for.body7.36: +; CHECK-UNROLL-NEXT: [[TMP24:%.*]] = load ptr, ptr [[ARRAYIDX_3]], align 8 +; CHECK-UNROLL-NEXT: [[TMP25:%.*]] = load i32, ptr [[TMP24]], align 4 +; CHECK-UNROLL-NEXT: [[TMP26:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_35:%.*]] = add nsw i32 [[TMP26]], [[TMP25]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_35]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_3:%.*]] +; CHECK-UNROLL: for.body7.1.3: +; CHECK-UNROLL-NEXT: [[TMP27:%.*]] = load ptr, ptr [[ARRAYIDX_3]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_3:%.*]] = getelementptr inbounds i32, ptr [[TMP27]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP28:%.*]] = load i32, ptr [[ARRAYIDX11_1_3]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP29:%.*]] = load i32, ptr [[ARRAYIDX13_1_3]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_3:%.*]] = add nsw i32 [[TMP29]], [[TMP28]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_3]], ptr [[ARRAYIDX13_1_3]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_3:%.*]] +; CHECK-UNROLL: for.body7.2.3: +; CHECK-UNROLL-NEXT: [[TMP30:%.*]] = load ptr, ptr [[ARRAYIDX_3]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_3:%.*]] = getelementptr inbounds i32, ptr [[TMP30]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX11_2_3]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX13_2_3]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_3:%.*]] = add nsw i32 [[TMP32]], [[TMP31]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_3]], ptr [[ARRAYIDX13_2_3]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_3:%.*]] +; CHECK-UNROLL: for.body7.3.3: +; CHECK-UNROLL-NEXT: [[TMP33:%.*]] = load ptr, ptr [[ARRAYIDX_3]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_3:%.*]] = getelementptr inbounds i32, ptr [[TMP33]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP34:%.*]] = load i32, ptr [[ARRAYIDX11_3_3]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP35:%.*]] = load i32, ptr [[ARRAYIDX13_3_3]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_3:%.*]] = add nsw i32 [[TMP35]], [[TMP34]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_3]], ptr [[ARRAYIDX13_3_3]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_3:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.3: +; CHECK-UNROLL-NEXT: [[CMP1_4:%.*]] = icmp eq i32 4, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_4]], label [[CLEANUP]], label [[IF_END_4:%.*]] +; CHECK-UNROLL: if.end.4: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 4 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_4:%.*]] +; CHECK-UNROLL: for.cond4.4: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_48:%.*]] +; CHECK-UNROLL: for.body7.48: +; CHECK-UNROLL-NEXT: [[TMP36:%.*]] = load ptr, ptr [[ARRAYIDX_4]], align 8 +; CHECK-UNROLL-NEXT: [[TMP37:%.*]] = load i32, ptr [[TMP36]], align 4 +; CHECK-UNROLL-NEXT: [[TMP38:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_47:%.*]] = add nsw i32 [[TMP38]], [[TMP37]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_47]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_4:%.*]] +; CHECK-UNROLL: for.body7.1.4: +; CHECK-UNROLL-NEXT: [[TMP39:%.*]] = load ptr, ptr [[ARRAYIDX_4]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_4:%.*]] = getelementptr inbounds i32, ptr [[TMP39]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP40:%.*]] = load i32, ptr [[ARRAYIDX11_1_4]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP41:%.*]] = load i32, ptr [[ARRAYIDX13_1_4]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_4:%.*]] = add nsw i32 [[TMP41]], [[TMP40]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_4]], ptr [[ARRAYIDX13_1_4]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_4:%.*]] +; CHECK-UNROLL: for.body7.2.4: +; CHECK-UNROLL-NEXT: [[TMP42:%.*]] = load ptr, ptr [[ARRAYIDX_4]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_4:%.*]] = getelementptr inbounds i32, ptr [[TMP42]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP43:%.*]] = load i32, ptr [[ARRAYIDX11_2_4]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP44:%.*]] = load i32, ptr [[ARRAYIDX13_2_4]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_4:%.*]] = add nsw i32 [[TMP44]], [[TMP43]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_4]], ptr [[ARRAYIDX13_2_4]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_4:%.*]] +; CHECK-UNROLL: for.body7.3.4: +; CHECK-UNROLL-NEXT: [[TMP45:%.*]] = load ptr, ptr [[ARRAYIDX_4]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_4:%.*]] = getelementptr inbounds i32, ptr [[TMP45]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP46:%.*]] = load i32, ptr [[ARRAYIDX11_3_4]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP47:%.*]] = load i32, ptr [[ARRAYIDX13_3_4]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_4:%.*]] = add nsw i32 [[TMP47]], [[TMP46]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_4]], ptr [[ARRAYIDX13_3_4]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_4:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.4: +; CHECK-UNROLL-NEXT: [[CMP1_5:%.*]] = icmp eq i32 5, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_5]], label [[CLEANUP]], label [[IF_END_5:%.*]] +; CHECK-UNROLL: if.end.5: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 5 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_5:%.*]] +; CHECK-UNROLL: for.cond4.5: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_5:%.*]] +; CHECK-UNROLL: for.body7.5: +; CHECK-UNROLL-NEXT: [[TMP48:%.*]] = load ptr, ptr [[ARRAYIDX_5]], align 8 +; CHECK-UNROLL-NEXT: [[TMP49:%.*]] = load i32, ptr [[TMP48]], align 4 +; CHECK-UNROLL-NEXT: [[TMP50:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_5:%.*]] = add nsw i32 [[TMP50]], [[TMP49]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_5]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_5:%.*]] +; CHECK-UNROLL: for.body7.1.5: +; CHECK-UNROLL-NEXT: [[TMP51:%.*]] = load ptr, ptr [[ARRAYIDX_5]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_5:%.*]] = getelementptr inbounds i32, ptr [[TMP51]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP52:%.*]] = load i32, ptr [[ARRAYIDX11_1_5]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP53:%.*]] = load i32, ptr [[ARRAYIDX13_1_5]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_5:%.*]] = add nsw i32 [[TMP53]], [[TMP52]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_5]], ptr [[ARRAYIDX13_1_5]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_5:%.*]] +; CHECK-UNROLL: for.body7.2.5: +; CHECK-UNROLL-NEXT: [[TMP54:%.*]] = load ptr, ptr [[ARRAYIDX_5]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_5:%.*]] = getelementptr inbounds i32, ptr [[TMP54]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP55:%.*]] = load i32, ptr [[ARRAYIDX11_2_5]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP56:%.*]] = load i32, ptr [[ARRAYIDX13_2_5]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_5:%.*]] = add nsw i32 [[TMP56]], [[TMP55]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_5]], ptr [[ARRAYIDX13_2_5]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_5:%.*]] +; CHECK-UNROLL: for.body7.3.5: +; CHECK-UNROLL-NEXT: [[TMP57:%.*]] = load ptr, ptr [[ARRAYIDX_5]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_5:%.*]] = getelementptr inbounds i32, ptr [[TMP57]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP58:%.*]] = load i32, ptr [[ARRAYIDX11_3_5]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_5:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP59:%.*]] = load i32, ptr [[ARRAYIDX13_3_5]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_5:%.*]] = add nsw i32 [[TMP59]], [[TMP58]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_5]], ptr [[ARRAYIDX13_3_5]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_5:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.5: +; CHECK-UNROLL-NEXT: [[CMP1_6:%.*]] = icmp eq i32 6, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_6]], label [[CLEANUP]], label [[IF_END_6:%.*]] +; CHECK-UNROLL: if.end.6: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 6 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_6:%.*]] +; CHECK-UNROLL: for.cond4.6: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_6:%.*]] +; CHECK-UNROLL: for.body7.6: +; CHECK-UNROLL-NEXT: [[TMP60:%.*]] = load ptr, ptr [[ARRAYIDX_6]], align 8 +; CHECK-UNROLL-NEXT: [[TMP61:%.*]] = load i32, ptr [[TMP60]], align 4 +; CHECK-UNROLL-NEXT: [[TMP62:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_6:%.*]] = add nsw i32 [[TMP62]], [[TMP61]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_6]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_6:%.*]] +; CHECK-UNROLL: for.body7.1.6: +; CHECK-UNROLL-NEXT: [[TMP63:%.*]] = load ptr, ptr [[ARRAYIDX_6]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_6:%.*]] = getelementptr inbounds i32, ptr [[TMP63]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP64:%.*]] = load i32, ptr [[ARRAYIDX11_1_6]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_6:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP65:%.*]] = load i32, ptr [[ARRAYIDX13_1_6]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_6:%.*]] = add nsw i32 [[TMP65]], [[TMP64]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_6]], ptr [[ARRAYIDX13_1_6]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_6:%.*]] +; CHECK-UNROLL: for.body7.2.6: +; CHECK-UNROLL-NEXT: [[TMP66:%.*]] = load ptr, ptr [[ARRAYIDX_6]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_6:%.*]] = getelementptr inbounds i32, ptr [[TMP66]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP67:%.*]] = load i32, ptr [[ARRAYIDX11_2_6]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_6:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP68:%.*]] = load i32, ptr [[ARRAYIDX13_2_6]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_6:%.*]] = add nsw i32 [[TMP68]], [[TMP67]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_6]], ptr [[ARRAYIDX13_2_6]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_6:%.*]] +; CHECK-UNROLL: for.body7.3.6: +; CHECK-UNROLL-NEXT: [[TMP69:%.*]] = load ptr, ptr [[ARRAYIDX_6]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_6:%.*]] = getelementptr inbounds i32, ptr [[TMP69]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP70:%.*]] = load i32, ptr [[ARRAYIDX11_3_6]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_6:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP71:%.*]] = load i32, ptr [[ARRAYIDX13_3_6]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_6:%.*]] = add nsw i32 [[TMP71]], [[TMP70]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_6]], ptr [[ARRAYIDX13_3_6]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_6:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.6: +; CHECK-UNROLL-NEXT: [[CMP1_7:%.*]] = icmp eq i32 7, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_7]], label [[CLEANUP]], label [[IF_END_7:%.*]] +; CHECK-UNROLL: if.end.7: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 7 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_7:%.*]] +; CHECK-UNROLL: for.cond4.7: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_7:%.*]] +; CHECK-UNROLL: for.body7.7: +; CHECK-UNROLL-NEXT: [[TMP72:%.*]] = load ptr, ptr [[ARRAYIDX_7]], align 8 +; CHECK-UNROLL-NEXT: [[TMP73:%.*]] = load i32, ptr [[TMP72]], align 4 +; CHECK-UNROLL-NEXT: [[TMP74:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_7:%.*]] = add nsw i32 [[TMP74]], [[TMP73]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_7]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_7:%.*]] +; CHECK-UNROLL: for.body7.1.7: +; CHECK-UNROLL-NEXT: [[TMP75:%.*]] = load ptr, ptr [[ARRAYIDX_7]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_7:%.*]] = getelementptr inbounds i32, ptr [[TMP75]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP76:%.*]] = load i32, ptr [[ARRAYIDX11_1_7]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_7:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP77:%.*]] = load i32, ptr [[ARRAYIDX13_1_7]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_7:%.*]] = add nsw i32 [[TMP77]], [[TMP76]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_7]], ptr [[ARRAYIDX13_1_7]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_7:%.*]] +; CHECK-UNROLL: for.body7.2.7: +; CHECK-UNROLL-NEXT: [[TMP78:%.*]] = load ptr, ptr [[ARRAYIDX_7]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_7:%.*]] = getelementptr inbounds i32, ptr [[TMP78]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP79:%.*]] = load i32, ptr [[ARRAYIDX11_2_7]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_7:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP80:%.*]] = load i32, ptr [[ARRAYIDX13_2_7]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_7:%.*]] = add nsw i32 [[TMP80]], [[TMP79]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_7]], ptr [[ARRAYIDX13_2_7]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_7:%.*]] +; CHECK-UNROLL: for.body7.3.7: +; CHECK-UNROLL-NEXT: [[TMP81:%.*]] = load ptr, ptr [[ARRAYIDX_7]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_7:%.*]] = getelementptr inbounds i32, ptr [[TMP81]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP82:%.*]] = load i32, ptr [[ARRAYIDX11_3_7]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_7:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP83:%.*]] = load i32, ptr [[ARRAYIDX13_3_7]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_7:%.*]] = add nsw i32 [[TMP83]], [[TMP82]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_7]], ptr [[ARRAYIDX13_3_7]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_7:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.7: +; CHECK-UNROLL-NEXT: [[CMP1_8:%.*]] = icmp eq i32 8, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_8]], label [[CLEANUP]], label [[IF_END_8:%.*]] +; CHECK-UNROLL: if.end.8: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_8:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 8 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_8:%.*]] +; CHECK-UNROLL: for.cond4.8: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_8:%.*]] +; CHECK-UNROLL: for.body7.8: +; CHECK-UNROLL-NEXT: [[TMP84:%.*]] = load ptr, ptr [[ARRAYIDX_8]], align 8 +; CHECK-UNROLL-NEXT: [[TMP85:%.*]] = load i32, ptr [[TMP84]], align 4 +; CHECK-UNROLL-NEXT: [[TMP86:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_8:%.*]] = add nsw i32 [[TMP86]], [[TMP85]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_8]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_8:%.*]] +; CHECK-UNROLL: for.body7.1.8: +; CHECK-UNROLL-NEXT: [[TMP87:%.*]] = load ptr, ptr [[ARRAYIDX_8]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_8:%.*]] = getelementptr inbounds i32, ptr [[TMP87]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP88:%.*]] = load i32, ptr [[ARRAYIDX11_1_8]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_8:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP89:%.*]] = load i32, ptr [[ARRAYIDX13_1_8]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_8:%.*]] = add nsw i32 [[TMP89]], [[TMP88]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_8]], ptr [[ARRAYIDX13_1_8]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_8:%.*]] +; CHECK-UNROLL: for.body7.2.8: +; CHECK-UNROLL-NEXT: [[TMP90:%.*]] = load ptr, ptr [[ARRAYIDX_8]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_8:%.*]] = getelementptr inbounds i32, ptr [[TMP90]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP91:%.*]] = load i32, ptr [[ARRAYIDX11_2_8]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_8:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP92:%.*]] = load i32, ptr [[ARRAYIDX13_2_8]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_8:%.*]] = add nsw i32 [[TMP92]], [[TMP91]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_8]], ptr [[ARRAYIDX13_2_8]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_8:%.*]] +; CHECK-UNROLL: for.body7.3.8: +; CHECK-UNROLL-NEXT: [[TMP93:%.*]] = load ptr, ptr [[ARRAYIDX_8]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_8:%.*]] = getelementptr inbounds i32, ptr [[TMP93]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP94:%.*]] = load i32, ptr [[ARRAYIDX11_3_8]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_8:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP95:%.*]] = load i32, ptr [[ARRAYIDX13_3_8]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_8:%.*]] = add nsw i32 [[TMP95]], [[TMP94]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_8]], ptr [[ARRAYIDX13_3_8]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_8:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.8: +; CHECK-UNROLL-NEXT: [[CMP1_9:%.*]] = icmp eq i32 9, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_9]], label [[CLEANUP]], label [[IF_END_9:%.*]] +; CHECK-UNROLL: if.end.9: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_9:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 9 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_9:%.*]] +; CHECK-UNROLL: for.cond4.9: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_9:%.*]] +; CHECK-UNROLL: for.body7.9: +; CHECK-UNROLL-NEXT: [[TMP96:%.*]] = load ptr, ptr [[ARRAYIDX_9]], align 8 +; CHECK-UNROLL-NEXT: [[TMP97:%.*]] = load i32, ptr [[TMP96]], align 4 +; CHECK-UNROLL-NEXT: [[TMP98:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_9:%.*]] = add nsw i32 [[TMP98]], [[TMP97]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_9]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_9:%.*]] +; CHECK-UNROLL: for.body7.1.9: +; CHECK-UNROLL-NEXT: [[TMP99:%.*]] = load ptr, ptr [[ARRAYIDX_9]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_9:%.*]] = getelementptr inbounds i32, ptr [[TMP99]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP100:%.*]] = load i32, ptr [[ARRAYIDX11_1_9]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_9:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP101:%.*]] = load i32, ptr [[ARRAYIDX13_1_9]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_9:%.*]] = add nsw i32 [[TMP101]], [[TMP100]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_9]], ptr [[ARRAYIDX13_1_9]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_9:%.*]] +; CHECK-UNROLL: for.body7.2.9: +; CHECK-UNROLL-NEXT: [[TMP102:%.*]] = load ptr, ptr [[ARRAYIDX_9]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_9:%.*]] = getelementptr inbounds i32, ptr [[TMP102]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP103:%.*]] = load i32, ptr [[ARRAYIDX11_2_9]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_9:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP104:%.*]] = load i32, ptr [[ARRAYIDX13_2_9]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_9:%.*]] = add nsw i32 [[TMP104]], [[TMP103]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_9]], ptr [[ARRAYIDX13_2_9]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_9:%.*]] +; CHECK-UNROLL: for.body7.3.9: +; CHECK-UNROLL-NEXT: [[TMP105:%.*]] = load ptr, ptr [[ARRAYIDX_9]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_9:%.*]] = getelementptr inbounds i32, ptr [[TMP105]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP106:%.*]] = load i32, ptr [[ARRAYIDX11_3_9]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_9:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP107:%.*]] = load i32, ptr [[ARRAYIDX13_3_9]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_9:%.*]] = add nsw i32 [[TMP107]], [[TMP106]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_9]], ptr [[ARRAYIDX13_3_9]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_9:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.9: +; CHECK-UNROLL-NEXT: [[CMP1_10:%.*]] = icmp eq i32 10, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_10]], label [[CLEANUP]], label [[IF_END_10:%.*]] +; CHECK-UNROLL: if.end.10: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_10:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 10 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_10:%.*]] +; CHECK-UNROLL: for.cond4.10: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_10:%.*]] +; CHECK-UNROLL: for.body7.10: +; CHECK-UNROLL-NEXT: [[TMP108:%.*]] = load ptr, ptr [[ARRAYIDX_10]], align 8 +; CHECK-UNROLL-NEXT: [[TMP109:%.*]] = load i32, ptr [[TMP108]], align 4 +; CHECK-UNROLL-NEXT: [[TMP110:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_10:%.*]] = add nsw i32 [[TMP110]], [[TMP109]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_10]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_10:%.*]] +; CHECK-UNROLL: for.body7.1.10: +; CHECK-UNROLL-NEXT: [[TMP111:%.*]] = load ptr, ptr [[ARRAYIDX_10]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_10:%.*]] = getelementptr inbounds i32, ptr [[TMP111]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP112:%.*]] = load i32, ptr [[ARRAYIDX11_1_10]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_10:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP113:%.*]] = load i32, ptr [[ARRAYIDX13_1_10]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_10:%.*]] = add nsw i32 [[TMP113]], [[TMP112]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_10]], ptr [[ARRAYIDX13_1_10]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_10:%.*]] +; CHECK-UNROLL: for.body7.2.10: +; CHECK-UNROLL-NEXT: [[TMP114:%.*]] = load ptr, ptr [[ARRAYIDX_10]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_10:%.*]] = getelementptr inbounds i32, ptr [[TMP114]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP115:%.*]] = load i32, ptr [[ARRAYIDX11_2_10]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_10:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP116:%.*]] = load i32, ptr [[ARRAYIDX13_2_10]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_10:%.*]] = add nsw i32 [[TMP116]], [[TMP115]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_10]], ptr [[ARRAYIDX13_2_10]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_10:%.*]] +; CHECK-UNROLL: for.body7.3.10: +; CHECK-UNROLL-NEXT: [[TMP117:%.*]] = load ptr, ptr [[ARRAYIDX_10]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_10:%.*]] = getelementptr inbounds i32, ptr [[TMP117]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP118:%.*]] = load i32, ptr [[ARRAYIDX11_3_10]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_10:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP119:%.*]] = load i32, ptr [[ARRAYIDX13_3_10]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_10:%.*]] = add nsw i32 [[TMP119]], [[TMP118]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_10]], ptr [[ARRAYIDX13_3_10]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_10:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.10: +; CHECK-UNROLL-NEXT: [[CMP1_11:%.*]] = icmp eq i32 11, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_11]], label [[CLEANUP]], label [[IF_END_11:%.*]] +; CHECK-UNROLL: if.end.11: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_11:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 11 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_11:%.*]] +; CHECK-UNROLL: for.cond4.11: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_11:%.*]] +; CHECK-UNROLL: for.body7.11: +; CHECK-UNROLL-NEXT: [[TMP120:%.*]] = load ptr, ptr [[ARRAYIDX_11]], align 8 +; CHECK-UNROLL-NEXT: [[TMP121:%.*]] = load i32, ptr [[TMP120]], align 4 +; CHECK-UNROLL-NEXT: [[TMP122:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_119:%.*]] = add nsw i32 [[TMP122]], [[TMP121]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_119]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_11:%.*]] +; CHECK-UNROLL: for.body7.1.11: +; CHECK-UNROLL-NEXT: [[TMP123:%.*]] = load ptr, ptr [[ARRAYIDX_11]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_11:%.*]] = getelementptr inbounds i32, ptr [[TMP123]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP124:%.*]] = load i32, ptr [[ARRAYIDX11_1_11]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_11:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP125:%.*]] = load i32, ptr [[ARRAYIDX13_1_11]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_11:%.*]] = add nsw i32 [[TMP125]], [[TMP124]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_11]], ptr [[ARRAYIDX13_1_11]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_11:%.*]] +; CHECK-UNROLL: for.body7.2.11: +; CHECK-UNROLL-NEXT: [[TMP126:%.*]] = load ptr, ptr [[ARRAYIDX_11]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_11:%.*]] = getelementptr inbounds i32, ptr [[TMP126]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP127:%.*]] = load i32, ptr [[ARRAYIDX11_2_11]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_11:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP128:%.*]] = load i32, ptr [[ARRAYIDX13_2_11]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_11:%.*]] = add nsw i32 [[TMP128]], [[TMP127]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_11]], ptr [[ARRAYIDX13_2_11]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_11:%.*]] +; CHECK-UNROLL: for.body7.3.11: +; CHECK-UNROLL-NEXT: [[TMP129:%.*]] = load ptr, ptr [[ARRAYIDX_11]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_11:%.*]] = getelementptr inbounds i32, ptr [[TMP129]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP130:%.*]] = load i32, ptr [[ARRAYIDX11_3_11]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_11:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP131:%.*]] = load i32, ptr [[ARRAYIDX13_3_11]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_11:%.*]] = add nsw i32 [[TMP131]], [[TMP130]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_11]], ptr [[ARRAYIDX13_3_11]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_11:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.11: +; CHECK-UNROLL-NEXT: [[CMP1_12:%.*]] = icmp eq i32 12, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_12]], label [[CLEANUP]], label [[IF_END_12:%.*]] +; CHECK-UNROLL: if.end.12: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_12:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 12 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_12:%.*]] +; CHECK-UNROLL: for.cond4.12: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1210:%.*]] +; CHECK-UNROLL: for.body7.1210: +; CHECK-UNROLL-NEXT: [[TMP132:%.*]] = load ptr, ptr [[ARRAYIDX_12]], align 8 +; CHECK-UNROLL-NEXT: [[TMP133:%.*]] = load i32, ptr [[TMP132]], align 4 +; CHECK-UNROLL-NEXT: [[TMP134:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_12:%.*]] = add nsw i32 [[TMP134]], [[TMP133]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_12]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_12:%.*]] +; CHECK-UNROLL: for.body7.1.12: +; CHECK-UNROLL-NEXT: [[TMP135:%.*]] = load ptr, ptr [[ARRAYIDX_12]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_12:%.*]] = getelementptr inbounds i32, ptr [[TMP135]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP136:%.*]] = load i32, ptr [[ARRAYIDX11_1_12]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_12:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP137:%.*]] = load i32, ptr [[ARRAYIDX13_1_12]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_12:%.*]] = add nsw i32 [[TMP137]], [[TMP136]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_12]], ptr [[ARRAYIDX13_1_12]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_12:%.*]] +; CHECK-UNROLL: for.body7.2.12: +; CHECK-UNROLL-NEXT: [[TMP138:%.*]] = load ptr, ptr [[ARRAYIDX_12]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_12:%.*]] = getelementptr inbounds i32, ptr [[TMP138]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP139:%.*]] = load i32, ptr [[ARRAYIDX11_2_12]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_12:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP140:%.*]] = load i32, ptr [[ARRAYIDX13_2_12]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_12:%.*]] = add nsw i32 [[TMP140]], [[TMP139]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_12]], ptr [[ARRAYIDX13_2_12]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_12:%.*]] +; CHECK-UNROLL: for.body7.3.12: +; CHECK-UNROLL-NEXT: [[TMP141:%.*]] = load ptr, ptr [[ARRAYIDX_12]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_12:%.*]] = getelementptr inbounds i32, ptr [[TMP141]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP142:%.*]] = load i32, ptr [[ARRAYIDX11_3_12]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_12:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP143:%.*]] = load i32, ptr [[ARRAYIDX13_3_12]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_12:%.*]] = add nsw i32 [[TMP143]], [[TMP142]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_12]], ptr [[ARRAYIDX13_3_12]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_12:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.12: +; CHECK-UNROLL-NEXT: [[CMP1_13:%.*]] = icmp eq i32 13, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_13]], label [[CLEANUP]], label [[IF_END_13:%.*]] +; CHECK-UNROLL: if.end.13: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_13:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 13 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_13:%.*]] +; CHECK-UNROLL: for.cond4.13: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_13:%.*]] +; CHECK-UNROLL: for.body7.13: +; CHECK-UNROLL-NEXT: [[TMP144:%.*]] = load ptr, ptr [[ARRAYIDX_13]], align 8 +; CHECK-UNROLL-NEXT: [[TMP145:%.*]] = load i32, ptr [[TMP144]], align 4 +; CHECK-UNROLL-NEXT: [[TMP146:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_13:%.*]] = add nsw i32 [[TMP146]], [[TMP145]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_13]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_13:%.*]] +; CHECK-UNROLL: for.body7.1.13: +; CHECK-UNROLL-NEXT: [[TMP147:%.*]] = load ptr, ptr [[ARRAYIDX_13]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_13:%.*]] = getelementptr inbounds i32, ptr [[TMP147]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP148:%.*]] = load i32, ptr [[ARRAYIDX11_1_13]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_13:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP149:%.*]] = load i32, ptr [[ARRAYIDX13_1_13]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_13:%.*]] = add nsw i32 [[TMP149]], [[TMP148]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_13]], ptr [[ARRAYIDX13_1_13]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_13:%.*]] +; CHECK-UNROLL: for.body7.2.13: +; CHECK-UNROLL-NEXT: [[TMP150:%.*]] = load ptr, ptr [[ARRAYIDX_13]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_13:%.*]] = getelementptr inbounds i32, ptr [[TMP150]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP151:%.*]] = load i32, ptr [[ARRAYIDX11_2_13]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_13:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP152:%.*]] = load i32, ptr [[ARRAYIDX13_2_13]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_13:%.*]] = add nsw i32 [[TMP152]], [[TMP151]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_13]], ptr [[ARRAYIDX13_2_13]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_13:%.*]] +; CHECK-UNROLL: for.body7.3.13: +; CHECK-UNROLL-NEXT: [[TMP153:%.*]] = load ptr, ptr [[ARRAYIDX_13]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_13:%.*]] = getelementptr inbounds i32, ptr [[TMP153]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP154:%.*]] = load i32, ptr [[ARRAYIDX11_3_13]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_13:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP155:%.*]] = load i32, ptr [[ARRAYIDX13_3_13]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_13:%.*]] = add nsw i32 [[TMP155]], [[TMP154]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_13]], ptr [[ARRAYIDX13_3_13]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_13:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.13: +; CHECK-UNROLL-NEXT: [[CMP1_14:%.*]] = icmp eq i32 14, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_14]], label [[CLEANUP]], label [[IF_END_14:%.*]] +; CHECK-UNROLL: if.end.14: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_14:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 14 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_14:%.*]] +; CHECK-UNROLL: for.cond4.14: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_14:%.*]] +; CHECK-UNROLL: for.body7.14: +; CHECK-UNROLL-NEXT: [[TMP156:%.*]] = load ptr, ptr [[ARRAYIDX_14]], align 8 +; CHECK-UNROLL-NEXT: [[TMP157:%.*]] = load i32, ptr [[TMP156]], align 4 +; CHECK-UNROLL-NEXT: [[TMP158:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_14:%.*]] = add nsw i32 [[TMP158]], [[TMP157]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_14]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_14:%.*]] +; CHECK-UNROLL: for.body7.1.14: +; CHECK-UNROLL-NEXT: [[TMP159:%.*]] = load ptr, ptr [[ARRAYIDX_14]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_14:%.*]] = getelementptr inbounds i32, ptr [[TMP159]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP160:%.*]] = load i32, ptr [[ARRAYIDX11_1_14]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_14:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP161:%.*]] = load i32, ptr [[ARRAYIDX13_1_14]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_14:%.*]] = add nsw i32 [[TMP161]], [[TMP160]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_14]], ptr [[ARRAYIDX13_1_14]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_14:%.*]] +; CHECK-UNROLL: for.body7.2.14: +; CHECK-UNROLL-NEXT: [[TMP162:%.*]] = load ptr, ptr [[ARRAYIDX_14]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_14:%.*]] = getelementptr inbounds i32, ptr [[TMP162]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP163:%.*]] = load i32, ptr [[ARRAYIDX11_2_14]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_14:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP164:%.*]] = load i32, ptr [[ARRAYIDX13_2_14]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_14:%.*]] = add nsw i32 [[TMP164]], [[TMP163]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_14]], ptr [[ARRAYIDX13_2_14]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_14:%.*]] +; CHECK-UNROLL: for.body7.3.14: +; CHECK-UNROLL-NEXT: [[TMP165:%.*]] = load ptr, ptr [[ARRAYIDX_14]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_14:%.*]] = getelementptr inbounds i32, ptr [[TMP165]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP166:%.*]] = load i32, ptr [[ARRAYIDX11_3_14]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_14:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP167:%.*]] = load i32, ptr [[ARRAYIDX13_3_14]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_14:%.*]] = add nsw i32 [[TMP167]], [[TMP166]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_14]], ptr [[ARRAYIDX13_3_14]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_14:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.14: +; CHECK-UNROLL-NEXT: [[CMP1_15:%.*]] = icmp eq i32 15, [[DIMS]] +; CHECK-UNROLL-NEXT: br i1 [[CMP1_15]], label [[CLEANUP]], label [[IF_END_15:%.*]] +; CHECK-UNROLL: if.end.15: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_15:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 15 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_15:%.*]] +; CHECK-UNROLL: for.cond4.15: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_15:%.*]] +; CHECK-UNROLL: for.body7.15: +; CHECK-UNROLL-NEXT: [[TMP168:%.*]] = load ptr, ptr [[ARRAYIDX_15]], align 8 +; CHECK-UNROLL-NEXT: [[TMP169:%.*]] = load i32, ptr [[TMP168]], align 4 +; CHECK-UNROLL-NEXT: [[TMP170:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_15:%.*]] = add nsw i32 [[TMP170]], [[TMP169]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_15]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_15:%.*]] +; CHECK-UNROLL: for.body7.1.15: +; CHECK-UNROLL-NEXT: [[TMP171:%.*]] = load ptr, ptr [[ARRAYIDX_15]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_15:%.*]] = getelementptr inbounds i32, ptr [[TMP171]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP172:%.*]] = load i32, ptr [[ARRAYIDX11_1_15]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_15:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP173:%.*]] = load i32, ptr [[ARRAYIDX13_1_15]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_15:%.*]] = add nsw i32 [[TMP173]], [[TMP172]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_15]], ptr [[ARRAYIDX13_1_15]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_15:%.*]] +; CHECK-UNROLL: for.body7.2.15: +; CHECK-UNROLL-NEXT: [[TMP174:%.*]] = load ptr, ptr [[ARRAYIDX_15]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_15:%.*]] = getelementptr inbounds i32, ptr [[TMP174]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP175:%.*]] = load i32, ptr [[ARRAYIDX11_2_15]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_15:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP176:%.*]] = load i32, ptr [[ARRAYIDX13_2_15]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_15:%.*]] = add nsw i32 [[TMP176]], [[TMP175]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_15]], ptr [[ARRAYIDX13_2_15]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_15:%.*]] +; CHECK-UNROLL: for.body7.3.15: +; CHECK-UNROLL-NEXT: [[TMP177:%.*]] = load ptr, ptr [[ARRAYIDX_15]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_15:%.*]] = getelementptr inbounds i32, ptr [[TMP177]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP178:%.*]] = load i32, ptr [[ARRAYIDX11_3_15]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_15:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP179:%.*]] = load i32, ptr [[ARRAYIDX13_3_15]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_15:%.*]] = add nsw i32 [[TMP179]], [[TMP178]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_15]], ptr [[ARRAYIDX13_3_15]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_15:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.15: +; CHECK-UNROLL-NEXT: br i1 true, label [[CLEANUP]], label [[IF_END_16:%.*]] +; CHECK-UNROLL: if.end.16: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_16:%.*]] = getelementptr inbounds ptr, ptr [[ARR]], i64 16 +; CHECK-UNROLL-NEXT: br label [[FOR_COND4_16:%.*]] +; CHECK-UNROLL: for.cond4.16: +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_16:%.*]] +; CHECK-UNROLL: for.body7.16: +; CHECK-UNROLL-NEXT: [[TMP180:%.*]] = load ptr, ptr [[ARRAYIDX_16]], align 8 +; CHECK-UNROLL-NEXT: [[TMP181:%.*]] = load i32, ptr [[TMP180]], align 4 +; CHECK-UNROLL-NEXT: [[TMP182:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_16:%.*]] = add nsw i32 [[TMP182]], [[TMP181]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_16]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1_16:%.*]] +; CHECK-UNROLL: for.body7.1.16: +; CHECK-UNROLL-NEXT: [[TMP183:%.*]] = load ptr, ptr [[ARRAYIDX_16]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1_16:%.*]] = getelementptr inbounds i32, ptr [[TMP183]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP184:%.*]] = load i32, ptr [[ARRAYIDX11_1_16]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1_16:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP185:%.*]] = load i32, ptr [[ARRAYIDX13_1_16]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1_16:%.*]] = add nsw i32 [[TMP185]], [[TMP184]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1_16]], ptr [[ARRAYIDX13_1_16]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2_16:%.*]] +; CHECK-UNROLL: for.body7.2.16: +; CHECK-UNROLL-NEXT: [[TMP186:%.*]] = load ptr, ptr [[ARRAYIDX_16]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2_16:%.*]] = getelementptr inbounds i32, ptr [[TMP186]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP187:%.*]] = load i32, ptr [[ARRAYIDX11_2_16]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2_16:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP188:%.*]] = load i32, ptr [[ARRAYIDX13_2_16]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2_16:%.*]] = add nsw i32 [[TMP188]], [[TMP187]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2_16]], ptr [[ARRAYIDX13_2_16]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3_16:%.*]] +; CHECK-UNROLL: for.body7.3.16: +; CHECK-UNROLL-NEXT: [[TMP189:%.*]] = load ptr, ptr [[ARRAYIDX_16]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3_16:%.*]] = getelementptr inbounds i32, ptr [[TMP189]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP190:%.*]] = load i32, ptr [[ARRAYIDX11_3_16]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3_16:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP191:%.*]] = load i32, ptr [[ARRAYIDX13_3_16]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3_16:%.*]] = add nsw i32 [[TMP191]], [[TMP190]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3_16]], ptr [[ARRAYIDX13_3_16]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6_16:%.*]] +; CHECK-UNROLL: for.cond.cleanup6.16: +; CHECK-UNROLL-NEXT: unreachable +; CHECK-UNROLL: for.body7: +; CHECK-UNROLL-NEXT: [[TMP192:%.*]] = load ptr, ptr [[ARR]], align 8 +; CHECK-UNROLL-NEXT: [[TMP193:%.*]] = load i32, ptr [[TMP192]], align 4 +; CHECK-UNROLL-NEXT: [[TMP194:%.*]] = load i32, ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14:%.*]] = add nsw i32 [[TMP194]], [[TMP193]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14]], ptr [[OUT]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_1:%.*]] +; CHECK-UNROLL: for.body7.1: +; CHECK-UNROLL-NEXT: [[TMP195:%.*]] = load ptr, ptr [[ARR]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_1:%.*]] = getelementptr inbounds i32, ptr [[TMP195]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP196:%.*]] = load i32, ptr [[ARRAYIDX11_1]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_1:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 1 +; CHECK-UNROLL-NEXT: [[TMP197:%.*]] = load i32, ptr [[ARRAYIDX13_1]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_1:%.*]] = add nsw i32 [[TMP197]], [[TMP196]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_1]], ptr [[ARRAYIDX13_1]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_2:%.*]] +; CHECK-UNROLL: for.body7.2: +; CHECK-UNROLL-NEXT: [[TMP198:%.*]] = load ptr, ptr [[ARR]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_2:%.*]] = getelementptr inbounds i32, ptr [[TMP198]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP199:%.*]] = load i32, ptr [[ARRAYIDX11_2]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_2:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 2 +; CHECK-UNROLL-NEXT: [[TMP200:%.*]] = load i32, ptr [[ARRAYIDX13_2]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_2:%.*]] = add nsw i32 [[TMP200]], [[TMP199]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_2]], ptr [[ARRAYIDX13_2]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br label [[FOR_BODY7_3:%.*]] +; CHECK-UNROLL: for.body7.3: +; CHECK-UNROLL-NEXT: [[TMP201:%.*]] = load ptr, ptr [[ARR]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_3:%.*]] = getelementptr inbounds i32, ptr [[TMP201]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP202:%.*]] = load i32, ptr [[ARRAYIDX11_3]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_3:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 3 +; CHECK-UNROLL-NEXT: [[TMP203:%.*]] = load i32, ptr [[ARRAYIDX13_3]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_3:%.*]] = add nsw i32 [[TMP203]], [[TMP202]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_3]], ptr [[ARRAYIDX13_3]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: br i1 false, label [[FOR_BODY7_4]], label [[FOR_COND_CLEANUP6:%.*]] +; CHECK-UNROLL: for.body7.4: +; CHECK-UNROLL-NEXT: [[ARRAYIDX_LCSSA:%.*]] = phi ptr [ [[ARR]], [[FOR_BODY7_3]] ], [ [[ARRAYIDX_1]], [[FOR_BODY7_3_1]] ], [ [[ARRAYIDX_2]], [[FOR_BODY7_3_2]] ], [ [[ARRAYIDX_3]], [[FOR_BODY7_3_3]] ], [ [[ARRAYIDX_4]], [[FOR_BODY7_3_4]] ], [ [[ARRAYIDX_5]], [[FOR_BODY7_3_5]] ], [ [[ARRAYIDX_6]], [[FOR_BODY7_3_6]] ], [ [[ARRAYIDX_7]], [[FOR_BODY7_3_7]] ], [ [[ARRAYIDX_8]], [[FOR_BODY7_3_8]] ], [ [[ARRAYIDX_9]], [[FOR_BODY7_3_9]] ], [ [[ARRAYIDX_10]], [[FOR_BODY7_3_10]] ], [ [[ARRAYIDX_11]], [[FOR_BODY7_3_11]] ], [ [[ARRAYIDX_12]], [[FOR_BODY7_3_12]] ], [ [[ARRAYIDX_13]], [[FOR_BODY7_3_13]] ], [ [[ARRAYIDX_14]], [[FOR_BODY7_3_14]] ], [ [[ARRAYIDX_15]], [[FOR_BODY7_3_15]] ], [ [[ARRAYIDX_16]], [[FOR_BODY7_3_16]] ] +; CHECK-UNROLL-NEXT: [[TMP204:%.*]] = load ptr, ptr [[ARRAYIDX_LCSSA]], align 8 +; CHECK-UNROLL-NEXT: [[ARRAYIDX11_4:%.*]] = getelementptr inbounds i32, ptr [[TMP204]], i64 4 +; CHECK-UNROLL-NEXT: [[TMP205:%.*]] = load i32, ptr [[ARRAYIDX11_4]], align 4 +; CHECK-UNROLL-NEXT: [[ARRAYIDX13_4:%.*]] = getelementptr inbounds i32, ptr [[OUT]], i64 4 +; CHECK-UNROLL-NEXT: [[TMP206:%.*]] = load i32, ptr [[ARRAYIDX13_4]], align 4 +; CHECK-UNROLL-NEXT: [[ADD14_4:%.*]] = add nsw i32 [[TMP206]], [[TMP205]] +; CHECK-UNROLL-NEXT: store i32 [[ADD14_4]], ptr [[ARRAYIDX13_4]], align 4 +; CHECK-UNROLL-NEXT: call void @_Z3barv() +; CHECK-UNROLL-NEXT: unreachable +; CHECK-UNROLL: cleanup: +; CHECK-UNROLL-NEXT: ret void +; +entry: + br label %for.cond + +for.cond: ; preds = %for.cond.cleanup6, %entry + %Dim.0 = phi i32 [ 0, %entry ], [ %inc16, %for.cond.cleanup6 ] + %Idx.addr.0 = phi i32 [ %Idx, %entry ], [ %add, %for.cond.cleanup6 ] + %cmp = icmp slt i32 %Dim.0, 16 + br i1 %cmp, label %for.body, label %for.cond.cleanup + +for.cond.cleanup: ; preds = %for.cond + br label %cleanup + +for.body: ; preds = %for.cond + %cmp1 = icmp eq i32 %Dim.0, %Dims + br i1 %cmp1, label %if.then, label %if.end + +if.then: ; preds = %for.body + br label %cleanup + +if.end: ; preds = %for.body + %idxprom = sext i32 %Dim.0 to i64 + %arrayidx = getelementptr inbounds ptr, ptr %Arr, i64 %idxprom + %0 = load ptr, ptr %arrayidx, align 8 + %idxprom2 = sext i32 %Idx.addr.0 to i64 + %arrayidx3 = getelementptr inbounds i32, ptr %0, i64 %idxprom2 + %1 = load i32, ptr %arrayidx3, align 4 + %add = add nsw i32 %1, 1 + br label %for.cond4 + +for.cond4: ; preds = %for.body7, %if.end + %arg.0 = phi i32 [ 0, %if.end ], [ %inc, %for.body7 ] + %cmp5 = icmp slt i32 %arg.0, 4 + br i1 %cmp5, label %for.body7, label %for.cond.cleanup6 + +for.cond.cleanup6: ; preds = %for.cond4 + %inc16 = add nsw i32 %Dim.0, 1 + br label %for.cond, !llvm.loop !0 + +for.body7: ; preds = %for.cond4 + %2 = load ptr, ptr %arrayidx, align 8 + %idxprom10 = sext i32 %arg.0 to i64 + %arrayidx11 = getelementptr inbounds i32, ptr %2, i64 %idxprom10 + %3 = load i32, ptr %arrayidx11, align 4 + %arrayidx13 = getelementptr inbounds i32, ptr %Out, i64 %idxprom10 + %4 = load i32, ptr %arrayidx13, align 4 + %add14 = add nsw i32 %4, %3 + store i32 %add14, ptr %arrayidx13, align 4 + call void @_Z3barv() + %inc = add nsw i32 %arg.0, 1 + br label %for.cond4, !llvm.loop !3 + +cleanup: ; preds = %if.then, %for.cond.cleanup + ret void +} + + declare void @_Z3barv() + +!0 = distinct !{!0, !1, !2} +!1 = !{!"llvm.loop.mustprogress"} +!2 = !{!"llvm.loop.unroll.enable"} +!3 = distinct !{!3, !1} +;. +; CHECK-CFG: [[LOOP0]] = distinct !{[[LOOP0]], [[META1:![0-9]+]], [[META2:![0-9]+]]} +; CHECK-CFG: [[META1]] = !{!"llvm.loop.mustprogress"} +; CHECK-CFG: [[META2]] = !{!"llvm.loop.unroll.enable"} +; CHECK-CFG: [[LOOP3]] = distinct !{[[LOOP3]], [[META1]]} +;. diff --git a/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll b/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll index cf875ccdc147..669306c8f3ab 100644 --- a/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll +++ b/llvm/test/Transforms/LoopUnroll/runtime-exit-phi-scev-invalidation.ll @@ -96,7 +96,7 @@ define void @pr56282() { ; CHECK: inner.2.preheader: ; CHECK-NEXT: br label [[INNER_2]] ; CHECK: inner.2: -; CHECK-NEXT: [[OUTER_IV_NEXT]] = add i64 [[OUTER_IV]], 1 +; CHECK-NEXT: [[OUTER_IV_NEXT]] = add nuw i64 [[OUTER_IV]], 1 ; CHECK-NEXT: br label [[OUTER_HEADER]] ; CHECK: exit: ; CHECK-NEXT: ret void diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll b/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll index 6b5d69d100dd..8c50d86489c9 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/eliminate-tail-predication.ll @@ -18,16 +18,16 @@ define void @f1(ptr %A) #0 { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0 -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer), ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0 +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer), ptr [[TMP8]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll b/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll index 98d55ae15c07..763b3e0bc829 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/gather-do-not-vectorize-addressing.ll @@ -54,6 +54,8 @@ define dso_local double @test(ptr nocapture noundef readonly %data, ptr nocaptur ; SVE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SVE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP3]] ; SVE-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]] +; SVE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; SVE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 ; SVE-NEXT: br label [[VECTOR_BODY:%.*]] ; SVE: vector.body: ; SVE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -66,8 +68,6 @@ define dso_local double @test(ptr nocapture noundef readonly %data, ptr nocaptur ; SVE-NEXT: [[TMP8:%.*]] = getelementptr inbounds double, ptr [[DATA:%.*]], [[TMP7]] ; SVE-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv2f64.nxv2p0( [[TMP8]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; SVE-NEXT: [[TMP9]] = fadd [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; SVE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; SVE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 ; SVE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; SVE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SVE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll index 28962dfba892..144b29d84198 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/masked-call.ll @@ -19,6 +19,8 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFNONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; TFNONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]] ; TFNONE: vector.body: ; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -27,8 +29,6 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP5:%.*]] = call @foo_vector( [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFNONE-NEXT: store [[TMP5]], ptr [[TMP6]], align 8 -; TFNONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; TFNONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; TFNONE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFNONE-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -60,6 +60,8 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 { ; TFCOMMON-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFCOMMON-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFCOMMON-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFCOMMON-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; TFCOMMON-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFCOMMON-NEXT: br label [[VECTOR_BODY:%.*]] ; TFCOMMON: vector.body: @@ -70,8 +72,6 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #4 { ; TFCOMMON-NEXT: [[TMP6:%.*]] = call @foo_vector( [[WIDE_MASKED_LOAD]], [[ACTIVE_LANE_MASK]]) ; TFCOMMON-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFCOMMON-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP6]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; TFCOMMON-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; TFCOMMON-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] ; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFCOMMON-NEXT: [[TMP10:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -111,6 +111,8 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFNONE-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; TFNONE-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]] ; TFNONE: vector.body: ; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -122,8 +124,6 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[PREDPHI:%.*]] = select [[TMP7]], zeroinitializer, [[TMP6]] ; TFNONE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] ; TFNONE-NEXT: store [[PREDPHI]], ptr [[TMP8]], align 8 -; TFNONE-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; TFNONE-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] ; TFNONE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFNONE-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -161,6 +161,8 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 { ; TFCOMMON-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFCOMMON-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFCOMMON-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFCOMMON-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; TFCOMMON-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 2 ; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFCOMMON-NEXT: br label [[VECTOR_BODY:%.*]] ; TFCOMMON: vector.body: @@ -177,8 +179,6 @@ define void @test_if_then(ptr noalias %a, ptr readnone %b) #4 { ; TFCOMMON-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] ; TFCOMMON-NEXT: [[TMP12:%.*]] = or [[TMP7]], [[TMP10]] ; TFCOMMON-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[PREDPHI]], ptr [[TMP11]], i32 8, [[TMP12]]) -; TFCOMMON-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; TFCOMMON-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 2 ; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]] ; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFCOMMON-NEXT: [[TMP15:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -229,6 +229,8 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFNONE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; TFNONE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 ; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]] ; TFNONE: vector.body: ; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -241,8 +243,6 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[PREDPHI:%.*]] = select [[TMP6]], [[TMP7]], [[TMP8]] ; TFNONE-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] ; TFNONE-NEXT: store [[PREDPHI]], ptr [[TMP9]], align 8 -; TFNONE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; TFNONE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 ; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; TFNONE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFNONE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -283,6 +283,8 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 { ; TFCOMMON-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFCOMMON-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFCOMMON-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFCOMMON-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; TFCOMMON-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2 ; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFCOMMON-NEXT: br label [[VECTOR_BODY:%.*]] ; TFCOMMON: vector.body: @@ -300,8 +302,6 @@ define void @test_widen_if_then_else(ptr noalias %a, ptr readnone %b) #4 { ; TFCOMMON-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] ; TFCOMMON-NEXT: [[TMP13:%.*]] = or [[TMP8]], [[TMP10]] ; TFCOMMON-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[PREDPHI]], ptr [[TMP12]], i32 8, [[TMP13]]) -; TFCOMMON-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; TFCOMMON-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2 ; TFCOMMON-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]] ; TFCOMMON-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFCOMMON-NEXT: [[TMP16:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -355,6 +355,8 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFNONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; TFNONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]] ; TFNONE: vector.body: ; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -363,8 +365,6 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP5:%.*]] = call @foo_vector_nomask( [[WIDE_LOAD]]) ; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFNONE-NEXT: store [[TMP5]], ptr [[TMP6]], align 8 -; TFNONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; TFNONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; TFNONE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFNONE-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -413,6 +413,8 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 { ; TFFALLBACK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFFALLBACK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFFALLBACK-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFFALLBACK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; TFFALLBACK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFFALLBACK-NEXT: br label [[VECTOR_BODY:%.*]] ; TFFALLBACK: vector.body: ; TFFALLBACK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -421,8 +423,6 @@ define void @test_widen_nomask(ptr noalias %a, ptr readnone %b) #4 { ; TFFALLBACK-NEXT: [[TMP5:%.*]] = call @foo_vector_nomask( [[WIDE_LOAD]]) ; TFFALLBACK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFFALLBACK-NEXT: store [[TMP5]], ptr [[TMP6]], align 8 -; TFFALLBACK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; TFFALLBACK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; TFFALLBACK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFFALLBACK-NEXT: br i1 [[TMP9]], label [[SCALAR_PH]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] @@ -475,6 +475,8 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFNONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; TFNONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]] ; TFNONE: vector.body: ; TFNONE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -483,8 +485,6 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFNONE-NEXT: [[TMP5:%.*]] = call @foo_vector_nomask( [[WIDE_LOAD]]) ; TFNONE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFNONE-NEXT: store [[TMP5]], ptr [[TMP6]], align 8 -; TFNONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; TFNONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; TFNONE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFNONE-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -516,6 +516,8 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFALWAYS-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFALWAYS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFALWAYS-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFALWAYS-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; TFALWAYS-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFALWAYS-NEXT: br label [[VECTOR_BODY:%.*]] ; TFALWAYS: vector.body: @@ -526,8 +528,6 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFALWAYS-NEXT: [[TMP6:%.*]] = call @foo_vector( [[WIDE_MASKED_LOAD]], [[ACTIVE_LANE_MASK]]) ; TFALWAYS-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFALWAYS-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP6]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; TFALWAYS-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; TFALWAYS-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TFALWAYS-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] ; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFALWAYS-NEXT: [[TMP10:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -546,6 +546,8 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFFALLBACK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFFALLBACK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFFALLBACK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFFALLBACK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; TFFALLBACK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFFALLBACK-NEXT: br label [[VECTOR_BODY:%.*]] ; TFFALLBACK: vector.body: @@ -556,8 +558,6 @@ define void @test_widen_optmask(ptr noalias %a, ptr readnone %b) #4 { ; TFFALLBACK-NEXT: [[TMP6:%.*]] = call @foo_vector( [[WIDE_MASKED_LOAD]], [[ACTIVE_LANE_MASK]]) ; TFFALLBACK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFFALLBACK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP6]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; TFFALLBACK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; TFFALLBACK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] ; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFFALLBACK-NEXT: [[TMP10:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -599,6 +599,8 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFNONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; TFNONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; TFNONE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; TFNONE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; TFNONE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 ; TFNONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, double [[M:%.*]], i64 0 ; TFNONE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; TFNONE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -613,8 +615,6 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFNONE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] ; TFNONE-NEXT: store [[TMP7]], ptr [[TMP8]], align 8 ; TFNONE-NEXT: [[TMP9]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], [[TMP5]]) -; TFNONE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; TFNONE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 ; TFNONE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; TFNONE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TFNONE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -651,6 +651,8 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFALWAYS-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFALWAYS-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFALWAYS-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFALWAYS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; TFALWAYS-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 ; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFALWAYS-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, double [[M:%.*]], i64 0 ; TFALWAYS-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer @@ -668,8 +670,6 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFALWAYS-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP8]], ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]]) ; TFALWAYS-NEXT: [[TMP10:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP6]], shufflevector ( insertelement ( poison, double -0.000000e+00, i64 0), poison, zeroinitializer) ; TFALWAYS-NEXT: [[TMP11]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], [[TMP10]]) -; TFALWAYS-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; TFALWAYS-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 ; TFALWAYS-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] ; TFALWAYS-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFALWAYS-NEXT: [[TMP14:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -688,6 +688,8 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFFALLBACK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TFFALLBACK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TFFALLBACK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TFFALLBACK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; TFFALLBACK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 ; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; TFFALLBACK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, double [[M:%.*]], i64 0 ; TFFALLBACK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer @@ -705,8 +707,6 @@ define double @test_widen_fmuladd_and_call(ptr noalias %a, ptr readnone %b, doub ; TFFALLBACK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP8]], ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]]) ; TFFALLBACK-NEXT: [[TMP10:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP6]], shufflevector ( insertelement ( poison, double -0.000000e+00, i64 0), poison, zeroinitializer) ; TFFALLBACK-NEXT: [[TMP11]] = call double @llvm.vector.reduce.fadd.nxv2f64(double [[VEC_PHI]], [[TMP10]]) -; TFFALLBACK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; TFFALLBACK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 ; TFFALLBACK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] ; TFFALLBACK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; TFFALLBACK-NEXT: [[TMP14:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll index ca3c935669e9..59a1e108b92f 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/outer_loop_prefer_scalable.ll @@ -18,6 +18,8 @@ define void @foo() { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 ; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -49,8 +51,6 @@ define void @foo() { ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4f32.nxv4p0( [[VEC_PHI5]], [[TMP10]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: [[TMP16:%.*]] = add nuw nsw [[VEC_IND]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq [[TMP16]], shufflevector ( insertelement ( poison, i64 1024, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll index 3f6d1b70f891..9bb941739852 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/pr60831-sve-inv-store-crash.ll @@ -17,28 +17,28 @@ define void @test_invar_gep(ptr %dst) #0 { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 100, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 100, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i64() +; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[INDEX]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer -; CHECK-NEXT: [[TMP5:%.*]] = add zeroinitializer, [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP7:%.*]] = add [[DOTSPLAT]], [[TMP6]] -; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 3 -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 0 -; CHECK-NEXT: [[TMP13:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP14:%.*]] = mul i32 [[TMP13]], 4 -; CHECK-NEXT: [[TMP15:%.*]] = sub i32 [[TMP14]], 1 -; CHECK-NEXT: [[TMP16:%.*]] = extractelement [[TMP7]], i32 [[TMP15]] -; CHECK-NEXT: store i64 [[TMP16]], ptr [[TMP12]], align 1 -; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[TMP17]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]] +; CHECK-NEXT: [[TMP7:%.*]] = add zeroinitializer, [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = mul [[TMP7]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP9:%.*]] = add [[DOTSPLAT]], [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 0 +; CHECK-NEXT: [[TMP15:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP16:%.*]] = mul i32 [[TMP15]], 4 +; CHECK-NEXT: [[TMP17:%.*]] = sub i32 [[TMP16]], 1 +; CHECK-NEXT: [[TMP18:%.*]] = extractelement [[TMP9]], i32 [[TMP17]] +; CHECK-NEXT: store i64 [[TMP18]], ptr [[TMP14]], align 1 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll index db97dc679673..a3c108bca140 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-avoid-scalarization.ll @@ -24,33 +24,33 @@ define void @test_no_scalarization(ptr %a, ptr noalias %b, i32 %idx, i32 %n) #0 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[TMP1]], [[TMP5]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP1]], [[N_MOD_VF]] ; CHECK-NEXT: [[IND_END:%.*]] = add i32 [[IDX]], [[N_VEC]] +; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP7:%.*]] = mul i32 [[TMP6]], 2 ; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[IDX]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv2i32() -; CHECK-NEXT: [[TMP7:%.*]] = add [[TMP6]], zeroinitializer -; CHECK-NEXT: [[TMP8:%.*]] = mul [[TMP7]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[INDUCTION:%.*]] = add [[DOTSPLAT]], [[TMP8]] -; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 2 -; CHECK-NEXT: [[TMP11:%.*]] = mul i32 1, [[TMP10]] -; CHECK-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP11]], i64 0 +; CHECK-NEXT: [[TMP8:%.*]] = call @llvm.experimental.stepvector.nxv2i32() +; CHECK-NEXT: [[TMP9:%.*]] = add [[TMP8]], zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = mul [[TMP9]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[INDUCTION:%.*]] = add [[DOTSPLAT]], [[TMP10]] +; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP12:%.*]] = mul i32 [[TMP11]], 2 +; CHECK-NEXT: [[TMP13:%.*]] = mul i32 1, [[TMP12]] +; CHECK-NEXT: [[DOTSPLATINSERT1:%.*]] = insertelement poison, i32 [[TMP13]], i64 0 ; CHECK-NEXT: [[DOTSPLAT2:%.*]] = shufflevector [[DOTSPLATINSERT1]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[OFFSET_IDX:%.*]] = add i32 [[IDX]], [[INDEX]] -; CHECK-NEXT: [[TMP12:%.*]] = add i32 [[OFFSET_IDX]], 0 -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i64, ptr [[A:%.*]], [[VEC_IND]] -; CHECK-NEXT: [[TMP14:%.*]] = extractelement [[TMP13]], i32 0 -; CHECK-NEXT: [[TMP15:%.*]] = getelementptr double, ptr [[TMP14]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP15]], align 8 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i64, ptr [[B:%.*]], i32 [[TMP12]] +; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[OFFSET_IDX]], 0 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr i64, ptr [[A:%.*]], [[VEC_IND]] +; CHECK-NEXT: [[TMP16:%.*]] = extractelement [[TMP15]], i32 0 ; CHECK-NEXT: [[TMP17:%.*]] = getelementptr double, ptr [[TMP16]], i32 0 -; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP17]], align 8 -; CHECK-NEXT: [[TMP18:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP19:%.*]] = mul i32 [[TMP18]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP19]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP17]], align 8 +; CHECK-NEXT: [[TMP18:%.*]] = getelementptr i64, ptr [[B:%.*]], i32 [[TMP14]] +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr double, ptr [[TMP18]], i32 0 +; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP19]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT2]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -58,7 +58,7 @@ define void @test_no_scalarization(ptr %a, ptr noalias %b, i32 %idx, i32 %n) #0 ; CHECK-NEXT: [[TMP21:%.*]] = call i32 @llvm.vscale.i32() ; CHECK-NEXT: [[TMP22:%.*]] = mul i32 [[TMP21]], 2 ; CHECK-NEXT: [[TMP23:%.*]] = sub i32 [[TMP22]], 1 -; CHECK-NEXT: [[TMP24:%.*]] = extractelement [[TMP13]], i32 [[TMP23]] +; CHECK-NEXT: [[TMP24:%.*]] = extractelement [[TMP15]], i32 [[TMP23]] ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[TMP1]], [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[L_EXIT:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll index 52feff81914a..43220d582c4c 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-reduction-inloop-cond.ll @@ -14,24 +14,24 @@ define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP12:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = fcmp une [[WIDE_LOAD]], shufflevector ( insertelement ( poison, float 2.000000e+00, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[TMP8]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP9]], i32 4, [[TMP7]], poison) -; CHECK-NEXT: [[TMP10:%.*]] = select fast [[TMP7]], [[WIDE_MASKED_LOAD]], zeroinitializer -; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, [[TMP10]]) -; CHECK-NEXT: [[TMP12]] = fadd fast float [[TMP11]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP14:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 4 +; CHECK-NEXT: [[TMP9:%.*]] = fcmp une [[WIDE_LOAD]], shufflevector ( insertelement ( poison, float 2.000000e+00, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr float, ptr [[TMP10]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP11]], i32 4, [[TMP9]], poison) +; CHECK-NEXT: [[TMP12:%.*]] = select fast [[TMP9]], [[WIDE_MASKED_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fadd.nxv4f32(float -0.000000e+00, [[TMP12]]) +; CHECK-NEXT: [[TMP14]] = fadd fast float [[TMP13]], [[VEC_PHI]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -39,7 +39,7 @@ define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 1.000000e+00, [[ENTRY]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 1.000000e+00, [[ENTRY]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[INDVARS:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[INDVARS_NEXT:%.*]], [[FOR_INC:%.*]] ] @@ -57,9 +57,9 @@ define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture ; CHECK-NEXT: [[RES]] = phi float [ [[FADD]], [[IF_THEN]] ], [ [[RDX]], [[FOR_BODY]] ] ; CHECK-NEXT: [[INDVARS_NEXT]] = add nuw nsw i64 [[INDVARS]], 1 ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: for.end: -; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP12]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP14]], [[MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret float [[RES_LCSSA]] ; entry: @@ -101,25 +101,25 @@ define float @cond_cmp_sel(ptr noalias %a, ptr noalias %cond, i64 %N) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[RDX_MINMAX_SELECT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = fcmp une [[WIDE_LOAD]], shufflevector ( insertelement ( poison, float 3.000000e+00, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr float, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr float, ptr [[TMP8]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP9]], i32 4, [[TMP7]], poison) -; CHECK-NEXT: [[TMP10:%.*]] = select fast [[TMP7]], [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, float 0x7FF0000000000000, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP11:%.*]] = call fast float @llvm.vector.reduce.fmin.nxv4f32( [[TMP10]]) -; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp fast olt float [[TMP11]], [[VEC_PHI]] -; CHECK-NEXT: [[RDX_MINMAX_SELECT]] = select fast i1 [[RDX_MINMAX_CMP]], float [[TMP11]], float [[VEC_PHI]] -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 4 +; CHECK-NEXT: [[TMP9:%.*]] = fcmp une [[WIDE_LOAD]], shufflevector ( insertelement ( poison, float 3.000000e+00, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr float, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr float, ptr [[TMP10]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP11]], i32 4, [[TMP9]], poison) +; CHECK-NEXT: [[TMP12:%.*]] = select fast [[TMP9]], [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, float 0x7FF0000000000000, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP13:%.*]] = call fast float @llvm.vector.reduce.fmin.nxv4f32( [[TMP12]]) +; CHECK-NEXT: [[RDX_MINMAX_CMP:%.*]] = fcmp fast olt float [[TMP13]], [[VEC_PHI]] +; CHECK-NEXT: [[RDX_MINMAX_SELECT]] = select fast i1 [[RDX_MINMAX_CMP]], float [[TMP13]], float [[VEC_PHI]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll index 2e994838ff24..cdc50c57b947 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/scalable-strict-fadd.ll @@ -42,6 +42,8 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 8 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -51,8 +53,6 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i32 0 ; CHECK-UNORDERED-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; CHECK-UNORDERED-NEXT: [[TMP7]] = fadd [[WIDE_LOAD]], [[VEC_PHI]] -; CHECK-UNORDERED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; CHECK-UNORDERED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -89,6 +89,8 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 8 ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-ORDERED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -98,8 +100,6 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[TMP5]], i32 0 ; CHECK-ORDERED-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; CHECK-ORDERED-NEXT: [[TMP7]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], [[WIDE_LOAD]]) -; CHECK-ORDERED-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; CHECK-ORDERED-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -136,6 +136,8 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP4]] ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 8 ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 8 ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -153,8 +155,6 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv8f32.p0(ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-ORDERED-TF-NEXT: [[TMP13:%.*]] = select [[ACTIVE_LANE_MASK]], [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[VEC_PHI]], [[TMP13]]) -; CHECK-ORDERED-TF-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 8 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP16]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv8i1.i64(i64 [[INDEX]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -230,6 +230,8 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP38:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP39:%.*]] = mul i64 [[TMP38]], 32 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -275,8 +277,6 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP35]] = fadd [[WIDE_LOAD4]], [[VEC_PHI1]] ; CHECK-UNORDERED-NEXT: [[TMP36]] = fadd [[WIDE_LOAD5]], [[VEC_PHI2]] ; CHECK-UNORDERED-NEXT: [[TMP37]] = fadd [[WIDE_LOAD6]], [[VEC_PHI3]] -; CHECK-UNORDERED-NEXT: [[TMP38:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP39:%.*]] = mul i64 [[TMP38]], 32 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP39]] ; CHECK-UNORDERED-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP40]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -316,6 +316,8 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-ORDERED-NEXT: [[TMP38:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP39:%.*]] = mul i64 [[TMP38]], 32 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -358,8 +360,6 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP35:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP34]], [[WIDE_LOAD1]]) ; CHECK-ORDERED-NEXT: [[TMP36:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP35]], [[WIDE_LOAD2]]) ; CHECK-ORDERED-NEXT: [[TMP37]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP36]], [[WIDE_LOAD3]]) -; CHECK-ORDERED-NEXT: [[TMP38:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP39:%.*]] = mul i64 [[TMP38]], 32 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP39]] ; CHECK-ORDERED-NEXT: [[TMP40:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP40]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -396,6 +396,8 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP4]] ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-ORDERED-TF-NEXT: [[TMP69:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP70:%.*]] = mul i64 [[TMP69]], 32 ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 32 ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -479,8 +481,6 @@ define float @fadd_strict_unroll(ptr noalias nocapture readonly %a, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP66:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP64]], [[TMP65]]) ; CHECK-ORDERED-TF-NEXT: [[TMP67:%.*]] = select [[ACTIVE_LANE_MASK8]], [[WIDE_MASKED_LOAD11]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP68]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP66]], [[TMP67]]) -; CHECK-ORDERED-TF-NEXT: [[TMP69:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP70:%.*]] = mul i64 [[TMP69]], 32 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP70]] ; CHECK-ORDERED-TF-NEXT: [[TMP71:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP72:%.*]] = mul i64 [[TMP71]], 8 @@ -589,6 +589,8 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] ; CHECK-UNORDERED-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 +; CHECK-UNORDERED-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 ; CHECK-UNORDERED-NEXT: [[TMP7:%.*]] = insertelement shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer), float [[A2]], i32 0 ; CHECK-UNORDERED-NEXT: [[TMP8:%.*]] = insertelement shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer), float [[A1]], i32 0 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] @@ -606,8 +608,6 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-UNORDERED-NEXT: [[TMP13:%.*]] = extractvalue { , } [[STRIDED_VEC]], 1 ; CHECK-UNORDERED-NEXT: [[TMP14]] = fadd [[TMP12]], [[VEC_PHI1]] ; CHECK-UNORDERED-NEXT: [[TMP15]] = fadd [[TMP13]], [[VEC_PHI]] -; CHECK-UNORDERED-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] ; CHECK-UNORDERED-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -661,6 +661,8 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] ; CHECK-ORDERED-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 +; CHECK-ORDERED-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 4 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -676,8 +678,6 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-ORDERED-NEXT: [[TMP11:%.*]] = extractvalue { , } [[STRIDED_VEC]], 1 ; CHECK-ORDERED-NEXT: [[TMP12]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP11]]) ; CHECK-ORDERED-NEXT: [[TMP13]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], [[TMP10]]) -; CHECK-ORDERED-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 4 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; CHECK-ORDERED-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -730,6 +730,8 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP4]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] ; CHECK-ORDERED-TF-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 2 +; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4 ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; CHECK-ORDERED-TF-NEXT: [[TMP10:%.*]] = sub i64 [[TMP2]], [[TMP9]] @@ -755,8 +757,6 @@ define void @fadd_strict_interleave(ptr noalias nocapture readonly %a, ptr noali ; CHECK-ORDERED-TF-NEXT: [[TMP19]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP18]]) ; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP16]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP21]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI1]], [[TMP20]]) -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP23]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP12]]) ; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -867,6 +867,8 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-UNORDERED-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 4 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -880,8 +882,6 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-UNORDERED-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP9]], align 4 ; CHECK-UNORDERED-NEXT: [[TMP10:%.*]] = fadd [[WIDE_LOAD]], [[WIDE_LOAD1]] ; CHECK-UNORDERED-NEXT: [[TMP11]] = fadd [[VEC_PHI]], [[TMP10]] -; CHECK-UNORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] ; CHECK-UNORDERED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -929,6 +929,8 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-ORDERED-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 4 ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP4]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-ORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -942,8 +944,6 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-ORDERED-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP9]], align 4 ; CHECK-ORDERED-NEXT: [[TMP10:%.*]] = fadd [[WIDE_LOAD]], [[WIDE_LOAD1]] ; CHECK-ORDERED-NEXT: [[TMP11]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP10]]) -; CHECK-ORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] ; CHECK-ORDERED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -991,6 +991,8 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-ORDERED-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP5]] ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP2]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 4 ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 ; CHECK-ORDERED-TF-NEXT: [[TMP8:%.*]] = sub i64 [[N]], [[TMP7]] @@ -1012,8 +1014,6 @@ define float @fadd_of_sum(ptr noalias nocapture readonly %a, ptr noalias nocaptu ; CHECK-ORDERED-TF-NEXT: [[TMP16:%.*]] = fadd [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]] ; CHECK-ORDERED-TF-NEXT: [[TMP17:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP16]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP18]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP17]]) -; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 4 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP20]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP10]]) ; CHECK-ORDERED-TF-NEXT: [[TMP21:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -1110,6 +1110,8 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no ; CHECK-UNORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1125,8 +1127,6 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no ; CHECK-UNORDERED-NEXT: [[TMP10:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-UNORDERED-NEXT: [[PREDPHI:%.*]] = select [[TMP10]], shufflevector ( insertelement ( poison, float 3.000000e+00, i64 0), poison, zeroinitializer), [[WIDE_MASKED_LOAD]] ; CHECK-UNORDERED-NEXT: [[TMP11]] = fadd [[VEC_PHI]], [[PREDPHI]] -; CHECK-UNORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] ; CHECK-UNORDERED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -1171,6 +1171,8 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no ; CHECK-ORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-ORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1186,8 +1188,6 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no ; CHECK-ORDERED-NEXT: [[TMP10:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-NEXT: [[PREDPHI:%.*]] = select [[TMP10]], shufflevector ( insertelement ( poison, float 3.000000e+00, i64 0), poison, zeroinitializer), [[WIDE_MASKED_LOAD]] ; CHECK-ORDERED-NEXT: [[TMP11]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[PREDPHI]]) -; CHECK-ORDERED-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] ; CHECK-ORDERED-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -1232,6 +1232,8 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no ; CHECK-ORDERED-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP4]] ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4 ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -1258,8 +1260,6 @@ define float @fadd_conditional(ptr noalias nocapture readonly %a, ptr noalias no ; CHECK-ORDERED-TF-NEXT: [[TMP19:%.*]] = or [[TMP15]], [[TMP18]] ; CHECK-ORDERED-TF-NEXT: [[TMP20:%.*]] = select [[TMP19]], [[PREDPHI]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP21]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP20]]) -; CHECK-ORDERED-TF-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP23]] ; CHECK-ORDERED-TF-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]]) ; CHECK-ORDERED-TF-NEXT: [[TMP24:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -1358,6 +1358,8 @@ define float @fadd_multiple(ptr noalias nocapture %a, ptr noalias nocapture %b, ; CHECK-UNORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 8 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 8 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1371,8 +1373,6 @@ define float @fadd_multiple(ptr noalias nocapture %a, ptr noalias nocapture %b, ; CHECK-UNORDERED-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[TMP8]], i32 0 ; CHECK-UNORDERED-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP9]], align 4 ; CHECK-UNORDERED-NEXT: [[TMP10]] = fadd [[TMP7]], [[WIDE_LOAD1]] -; CHECK-UNORDERED-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 8 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] ; CHECK-UNORDERED-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -1498,6 +1498,8 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP52:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP53:%.*]] = mul i64 [[TMP52]], 32 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1561,8 +1563,6 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP49]] = call @llvm.fmuladd.nxv8f32( [[WIDE_LOAD4]], [[WIDE_LOAD8]], [[VEC_PHI1]]) ; CHECK-UNORDERED-NEXT: [[TMP50]] = call @llvm.fmuladd.nxv8f32( [[WIDE_LOAD5]], [[WIDE_LOAD9]], [[VEC_PHI2]]) ; CHECK-UNORDERED-NEXT: [[TMP51]] = call @llvm.fmuladd.nxv8f32( [[WIDE_LOAD6]], [[WIDE_LOAD10]], [[VEC_PHI3]]) -; CHECK-UNORDERED-NEXT: [[TMP52:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP53:%.*]] = mul i64 [[TMP52]], 32 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP53]] ; CHECK-UNORDERED-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP54]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] @@ -1604,6 +1604,8 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-ORDERED-NEXT: [[TMP56:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP57:%.*]] = mul i64 [[TMP56]], 32 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1668,8 +1670,6 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP53:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP52]], [[TMP49]]) ; CHECK-ORDERED-NEXT: [[TMP54:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP53]], [[TMP50]]) ; CHECK-ORDERED-NEXT: [[TMP55]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP54]], [[TMP51]]) -; CHECK-ORDERED-NEXT: [[TMP56:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP57:%.*]] = mul i64 [[TMP56]], 32 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP57]] ; CHECK-ORDERED-NEXT: [[TMP58:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP58]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] @@ -1708,6 +1708,8 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP4]] ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-ORDERED-TF-NEXT: [[TMP87:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP88:%.*]] = mul i64 [[TMP87]], 32 ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 32 ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -1813,8 +1815,6 @@ define float @fmuladd_strict(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP84:%.*]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP82]], [[TMP83]]) ; CHECK-ORDERED-TF-NEXT: [[TMP85:%.*]] = select [[ACTIVE_LANE_MASK8]], [[TMP78]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP86]] = call float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP84]], [[TMP85]]) -; CHECK-ORDERED-TF-NEXT: [[TMP87:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP88:%.*]] = mul i64 [[TMP87]], 32 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP88]] ; CHECK-ORDERED-TF-NEXT: [[TMP89:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP90:%.*]] = mul i64 [[TMP89]], 8 @@ -1912,6 +1912,8 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-UNORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-UNORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-UNORDERED-NEXT: [[TMP52:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-UNORDERED-NEXT: [[TMP53:%.*]] = mul i64 [[TMP52]], 32 ; CHECK-UNORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-UNORDERED: vector.body: ; CHECK-UNORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -1975,8 +1977,6 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-UNORDERED-NEXT: [[TMP49]] = call nnan @llvm.fmuladd.nxv8f32( [[WIDE_LOAD4]], [[WIDE_LOAD8]], [[VEC_PHI1]]) ; CHECK-UNORDERED-NEXT: [[TMP50]] = call nnan @llvm.fmuladd.nxv8f32( [[WIDE_LOAD5]], [[WIDE_LOAD9]], [[VEC_PHI2]]) ; CHECK-UNORDERED-NEXT: [[TMP51]] = call nnan @llvm.fmuladd.nxv8f32( [[WIDE_LOAD6]], [[WIDE_LOAD10]], [[VEC_PHI3]]) -; CHECK-UNORDERED-NEXT: [[TMP52:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-UNORDERED-NEXT: [[TMP53:%.*]] = mul i64 [[TMP52]], 32 ; CHECK-UNORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP53]] ; CHECK-UNORDERED-NEXT: [[TMP54:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-UNORDERED-NEXT: br i1 [[TMP54]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] @@ -2018,6 +2018,8 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-ORDERED-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-ORDERED-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-ORDERED-NEXT: [[TMP56:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-NEXT: [[TMP57:%.*]] = mul i64 [[TMP56]], 32 ; CHECK-ORDERED-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-ORDERED: vector.body: ; CHECK-ORDERED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -2082,8 +2084,6 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-NEXT: [[TMP53:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP52]], [[TMP49]]) ; CHECK-ORDERED-NEXT: [[TMP54:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP53]], [[TMP50]]) ; CHECK-ORDERED-NEXT: [[TMP55]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP54]], [[TMP51]]) -; CHECK-ORDERED-NEXT: [[TMP56:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-NEXT: [[TMP57:%.*]] = mul i64 [[TMP56]], 32 ; CHECK-ORDERED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP57]] ; CHECK-ORDERED-NEXT: [[TMP58:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-ORDERED-NEXT: br i1 [[TMP58]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] @@ -2122,6 +2122,8 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[N_RND_UP:%.*]] = add i64 [[N]], [[TMP4]] ; CHECK-ORDERED-TF-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-ORDERED-TF-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-ORDERED-TF-NEXT: [[TMP87:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-ORDERED-TF-NEXT: [[TMP88:%.*]] = mul i64 [[TMP87]], 32 ; CHECK-ORDERED-TF-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 32 ; CHECK-ORDERED-TF-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -2227,8 +2229,6 @@ define float @fmuladd_strict_fmf(ptr %a, ptr %b, i64 %n) #0 { ; CHECK-ORDERED-TF-NEXT: [[TMP84:%.*]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP82]], [[TMP83]]) ; CHECK-ORDERED-TF-NEXT: [[TMP85:%.*]] = select nnan [[ACTIVE_LANE_MASK8]], [[TMP78]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-ORDERED-TF-NEXT: [[TMP86]] = call nnan float @llvm.vector.reduce.fadd.nxv8f32(float [[TMP84]], [[TMP85]]) -; CHECK-ORDERED-TF-NEXT: [[TMP87:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-ORDERED-TF-NEXT: [[TMP88:%.*]] = mul i64 [[TMP87]], 32 ; CHECK-ORDERED-TF-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP88]] ; CHECK-ORDERED-TF-NEXT: [[TMP89:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-ORDERED-TF-NEXT: [[TMP90:%.*]] = mul i64 [[TMP89]], 8 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll index b3de02799f51..7848d0836d54 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-cond-inv-loads.ll @@ -13,21 +13,21 @@ define void @cond_inv_load_i32i32i16(ptr noalias nocapture %a, ptr noalias nocap ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[INV:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP3]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = icmp ne [[WIDE_LOAD]], zeroinitializer -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i16.nxv4p0( [[BROADCAST_SPLAT]], i32 2, [[TMP4]], poison) -; CHECK-NEXT: [[TMP5:%.*]] = sext [[WIDE_MASKED_GATHER]] to -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP5]], ptr [[TMP6]], i32 4, [[TMP4]]) -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP5]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = icmp ne [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i16.nxv4p0( [[BROADCAST_SPLAT]], i32 2, [[TMP6]], poison) +; CHECK-NEXT: [[TMP7:%.*]] = sext [[WIDE_MASKED_GATHER]] to +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP7]], ptr [[TMP8]], i32 4, [[TMP6]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -92,20 +92,20 @@ define void @cond_inv_load_f64f64f64(ptr noalias nocapture %a, ptr noalias nocap ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[INV:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[COND:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP3]], align 8 -; CHECK-NEXT: [[TMP4:%.*]] = fcmp ogt [[WIDE_LOAD]], shufflevector ( insertelement ( poison, double 4.000000e-01, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f64.nxv4p0( [[BROADCAST_SPLAT]], i32 8, [[TMP4]], poison) -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: call void @llvm.masked.store.nxv4f64.p0( [[WIDE_MASKED_GATHER]], ptr [[TMP5]], i32 8, [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[COND:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP5]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = fcmp ogt [[WIDE_LOAD]], shufflevector ( insertelement ( poison, double 4.000000e-01, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f64.nxv4p0( [[BROADCAST_SPLAT]], i32 8, [[TMP6]], poison) +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr double, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: call void @llvm.masked.store.nxv4f64.p0( [[WIDE_MASKED_GATHER]], ptr [[TMP7]], i32 8, [[TMP6]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: @@ -168,24 +168,24 @@ define void @invariant_load_cond(ptr noalias nocapture %a, ptr nocapture readonl ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 42 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[TMP3]], i64 0 +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 42 +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[TMP5]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = icmp ne [[WIDE_LOAD]], zeroinitializer -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP6]], i32 4, [[TMP5]], poison) -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[BROADCAST_SPLAT]], i32 4, [[TMP5]], poison) -; CHECK-NEXT: [[TMP7:%.*]] = add nsw [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_LOAD]] -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP7]], ptr [[TMP8]], i32 4, [[TMP5]]) -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = shl nuw nsw i64 [[TMP9]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP8]], i32 4, [[TMP7]], poison) +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[BROADCAST_SPLAT]], i32 4, [[TMP7]], poison) +; CHECK-NEXT: [[TMP9:%.*]] = add nsw [[WIDE_MASKED_GATHER]], [[WIDE_MASKED_LOAD]] +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP9]], ptr [[TMP10]], i32 4, [[TMP7]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll index 04af618014a7..982915fe5cc5 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-inloop-reductions.ll @@ -19,6 +19,8 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -42,8 +44,6 @@ define i64 @int_reduction_and(ptr noalias nocapture %a, i64 %N) { ; CHECK-NEXT: [[TMP17]] = and i64 [[TMP16]], [[VEC_PHI]] ; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vector.reduce.and.nxv2i64( [[WIDE_LOAD3]]) ; CHECK-NEXT: [[TMP19]] = and i64 [[TMP18]], [[VEC_PHI2]] -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP21]] ; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP22]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll index f84e7c0ea182..f48933a76709 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-reductions.ll @@ -19,6 +19,8 @@ define i64 @int_reduction_add(ptr %a, i64 %N) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -40,8 +42,6 @@ define i64 @int_reduction_add(ptr %a, i64 %N) { ; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP15]], align 8 ; CHECK-NEXT: [[TMP16]] = add [[WIDE_LOAD]], [[VEC_PHI]] ; CHECK-NEXT: [[TMP17]] = add [[WIDE_LOAD3]], [[VEC_PHI2]] -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll index 23fd5fe5e908..fd3d8d09fb39 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect-strict-reductions.ll @@ -19,29 +19,29 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 8 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 8 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[VECTOR_PH]] ], [ [[TMP17:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 0 -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 1 -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]] -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP12]], align 4 -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 -; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i64 [[TMP14]] -; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP15]], align 4 -; CHECK-NEXT: [[TMP16:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[WIDE_LOAD]]) -; CHECK-NEXT: [[TMP17]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[TMP16]], [[WIDE_LOAD2]]) -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[VECTOR_PH]] ], [ [[TMP19:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 +; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP11]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP14]], align 4 +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[TMP16]] +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP17]], align 4 +; CHECK-NEXT: [[TMP18:%.*]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[WIDE_LOAD]]) +; CHECK-NEXT: [[TMP19]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[TMP18]], [[WIDE_LOAD2]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -52,7 +52,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) { ; CHECK-NEXT: [[MIN_EPILOG_ITERS_CHECK:%.*]] = icmp ult i64 [[N_VEC_REMAINING]], 2 ; CHECK-NEXT: br i1 [[MIN_EPILOG_ITERS_CHECK]], label [[VEC_EPILOG_SCALAR_PH]], label [[VEC_EPILOG_PH]] ; CHECK: vec.epilog.ph: -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ], [ [[TMP17]], [[VEC_EPILOG_ITER_CHECK]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ], [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ] ; CHECK-NEXT: [[VEC_EPILOG_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[VECTOR_MAIN_LOOP_ITER_CHECK]] ] ; CHECK-NEXT: [[N_MOD_VF3:%.*]] = urem i64 [[N]], 2 ; CHECK-NEXT: [[N_VEC4:%.*]] = sub i64 [[N]], [[N_MOD_VF3]] @@ -73,7 +73,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) { ; CHECK-NEXT: br i1 [[CMP_N5]], label [[FOR_END]], label [[VEC_EPILOG_SCALAR_PH]] ; CHECK: vec.epilog.scalar.ph: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC4]], [[VEC_EPILOG_MIDDLE_BLOCK]] ], [ [[N_VEC]], [[VEC_EPILOG_ITER_CHECK]] ], [ 0, [[ITER_CHECK:%.*]] ] -; CHECK-NEXT: [[BC_MERGE_RDX10:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[ITER_CHECK]] ], [ [[TMP17]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[BC_MERGE_RDX10:%.*]] = phi float [ 0xFFFFFFFFE0000000, [[ITER_CHECK]] ], [ [[TMP19]], [[VEC_EPILOG_ITER_CHECK]] ], [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[VEC_EPILOG_SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_BODY]] ] @@ -85,7 +85,7 @@ define float @fadd_strict(ptr noalias nocapture readonly %a, i64 %n) { ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] ; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: for.end: -; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP17]], [[MIDDLE_BLOCK]] ], [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[ADD_LCSSA:%.*]] = phi float [ [[ADD]], [[FOR_BODY]] ], [ [[TMP19]], [[MIDDLE_BLOCK]] ], [ [[TMP24]], [[VEC_EPILOG_MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret float [[ADD_LCSSA]] ; entry: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll index 6982b3f0943c..d5f41aa440e5 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-epilog-vect.ll @@ -35,26 +35,26 @@ define void @main_vf_vscale_x_16(ptr %A) #0 { ; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 32 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP5]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 32 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 16 -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 -; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1 -; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP6]] -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP11]] -; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i32 0 -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP14]], align 1 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 16 -; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 [[TMP16]] -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP17]], align 1 -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 32 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] +; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 16 +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP10]], 0 +; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 1 +; CHECK-NEXT: [[TMP13:%.*]] = add i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP13]] +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i32 0 +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP16]], align 1 +; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[TMP17]], 16 +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i8, ptr [[TMP14]], i64 [[TMP18]] +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP19]], align 1 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -72,16 +72,16 @@ define void @main_vf_vscale_x_16(ptr %A) #0 { ; CHECK-NEXT: [[TMP24:%.*]] = mul i64 [[TMP23]], 8 ; CHECK-NEXT: [[N_MOD_VF2:%.*]] = urem i64 1024, [[TMP24]] ; CHECK-NEXT: [[N_VEC3:%.*]] = sub i64 1024, [[N_MOD_VF2]] +; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP26:%.*]] = mul i64 [[TMP25]], 8 ; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] ; CHECK: vec.epilog.vector.body: ; CHECK-NEXT: [[INDEX5:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT6:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP25:%.*]] = add i64 [[INDEX5]], 0 -; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP25]] -; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i8, ptr [[TMP26]], i32 0 -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP27]], align 1 -; CHECK-NEXT: [[TMP28:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP29:%.*]] = mul i64 [[TMP28]], 8 -; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX5]], [[TMP29]] +; CHECK-NEXT: [[TMP27:%.*]] = add i64 [[INDEX5]], 0 +; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP27]] +; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds i8, ptr [[TMP28]], i32 0 +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP29]], align 1 +; CHECK-NEXT: [[INDEX_NEXT6]] = add nuw i64 [[INDEX5]], [[TMP26]] ; CHECK-NEXT: [[TMP30:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC3]] ; CHECK-NEXT: br i1 [[TMP30]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: vec.epilog.middle.block: @@ -113,26 +113,26 @@ define void @main_vf_vscale_x_16(ptr %A) #0 { ; CHECK-VF8-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-VF8-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-VF8-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 32 ; CHECK-VF8-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-VF8: vector.body: ; CHECK-VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-VF8-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-VF8-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 16 -; CHECK-VF8-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 0 -; CHECK-VF8-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 1 -; CHECK-VF8-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]] -; CHECK-VF8-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-VF8-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP9]] -; CHECK-VF8-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i32 0 -; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP12]], align 1 -; CHECK-VF8-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 16 -; CHECK-VF8-NEXT: [[TMP15:%.*]] = getelementptr inbounds i8, ptr [[TMP10]], i64 [[TMP14]] -; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP15]], align 1 -; CHECK-VF8-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 32 -; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] +; CHECK-VF8-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-VF8-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 16 +; CHECK-VF8-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 +; CHECK-VF8-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1 +; CHECK-VF8-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-VF8-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-VF8-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[A]], i64 [[TMP11]] +; CHECK-VF8-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i32 0 +; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP14]], align 1 +; CHECK-VF8-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 16 +; CHECK-VF8-NEXT: [[TMP17:%.*]] = getelementptr inbounds i8, ptr [[TMP12]], i64 [[TMP16]] +; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer), ptr [[TMP17]], align 1 +; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-VF8-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-VF8-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK-VF8: middle.block: @@ -213,26 +213,26 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 0 -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 1 -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP9]] -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i32 0 -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP12]], align 1 -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 2 -; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i64 [[TMP14]] -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP15]], align 1 -; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 +; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP11]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0 +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP14]], align 1 +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 [[TMP16]] +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP17]], align 1 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: @@ -282,26 +282,26 @@ define void @main_vf_vscale_x_2(ptr %A) #0 vscale_range(8, 8) { ; CHECK-VF8-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-VF8-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-VF8-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-VF8-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-VF8: vector.body: ; CHECK-VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-VF8-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-VF8-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 -; CHECK-VF8-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 0 -; CHECK-VF8-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 1 -; CHECK-VF8-NEXT: [[TMP9:%.*]] = add i64 [[INDEX]], [[TMP8]] -; CHECK-VF8-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-VF8-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP9]] -; CHECK-VF8-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i32 0 -; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP12]], align 1 -; CHECK-VF8-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 2 -; CHECK-VF8-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP10]], i64 [[TMP14]] -; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP15]], align 1 -; CHECK-VF8-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 -; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] +; CHECK-VF8-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-VF8-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 +; CHECK-VF8-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 +; CHECK-VF8-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 1 +; CHECK-VF8-NEXT: [[TMP11:%.*]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-VF8-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-VF8-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP11]] +; CHECK-VF8-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0 +; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP14]], align 1 +; CHECK-VF8-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 +; CHECK-VF8-NEXT: [[TMP17:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i64 [[TMP16]] +; CHECK-VF8-NEXT: store shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer), ptr [[TMP17]], align 1 +; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-VF8-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-VF8-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK-VF8: middle.block: @@ -372,26 +372,26 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 { ; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 32 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP5]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 10000, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 32 ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[N_VEC]] ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]] -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 16 -; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 -; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]] -; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP10]] -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0 -; CHECK-NEXT: store zeroinitializer, ptr [[TMP11]], align 1 -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 16 -; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP13]] -; CHECK-NEXT: store zeroinitializer, ptr [[TMP14]], align 1 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 32 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] +; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 16 +; CHECK-NEXT: [[TMP11:%.*]] = add i64 [[TMP10]], 0 +; CHECK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[NEXT_GEP2:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP12]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0 +; CHECK-NEXT: store zeroinitializer, ptr [[TMP13]], align 1 +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 16 +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP15]] +; CHECK-NEXT: store zeroinitializer, ptr [[TMP16]], align 1 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: middle.block: @@ -412,16 +412,16 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 { ; CHECK-NEXT: [[N_MOD_VF3:%.*]] = urem i64 10000, [[TMP21]] ; CHECK-NEXT: [[N_VEC4:%.*]] = sub i64 10000, [[N_MOD_VF3]] ; CHECK-NEXT: [[IND_END6:%.*]] = getelementptr i8, ptr [[START]], i64 [[N_VEC4]] +; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 8 ; CHECK-NEXT: br label [[VEC_EPILOG_VECTOR_BODY:%.*]] ; CHECK: vec.epilog.vector.body: ; CHECK-NEXT: [[INDEX10:%.*]] = phi i64 [ [[VEC_EPILOG_RESUME_VAL]], [[VEC_EPILOG_PH]] ], [ [[INDEX_NEXT12:%.*]], [[VEC_EPILOG_VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP22:%.*]] = add i64 [[INDEX10]], 0 -; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP22]] -; CHECK-NEXT: [[TMP23:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i32 0 -; CHECK-NEXT: store zeroinitializer, ptr [[TMP23]], align 1 -; CHECK-NEXT: [[TMP24:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP25:%.*]] = mul i64 [[TMP24]], 8 -; CHECK-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX10]], [[TMP25]] +; CHECK-NEXT: [[TMP24:%.*]] = add i64 [[INDEX10]], 0 +; CHECK-NEXT: [[NEXT_GEP11:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP24]] +; CHECK-NEXT: [[TMP25:%.*]] = getelementptr i8, ptr [[NEXT_GEP11]], i32 0 +; CHECK-NEXT: store zeroinitializer, ptr [[TMP25]], align 1 +; CHECK-NEXT: [[INDEX_NEXT12]] = add nuw i64 [[INDEX10]], [[TMP23]] ; CHECK-NEXT: [[TMP26:%.*]] = icmp eq i64 [[INDEX_NEXT12]], [[N_VEC4]] ; CHECK-NEXT: br i1 [[TMP26]], label [[VEC_EPILOG_MIDDLE_BLOCK:%.*]], label [[VEC_EPILOG_VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] ; CHECK: vec.epilog.middle.block: @@ -455,26 +455,26 @@ define void @test_pr57912_pointer_induction(ptr %start) #0 { ; CHECK-VF8-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 32 ; CHECK-VF8-NEXT: [[N_MOD_VF:%.*]] = urem i64 10000, [[TMP3]] ; CHECK-VF8-NEXT: [[N_VEC:%.*]] = sub i64 10000, [[N_MOD_VF]] +; CHECK-VF8-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 32 ; CHECK-VF8-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[N_VEC]] ; CHECK-VF8-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK-VF8: vector.body: ; CHECK-VF8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-VF8-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-VF8-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP4]] -; CHECK-VF8-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 16 -; CHECK-VF8-NEXT: [[TMP7:%.*]] = add i64 [[TMP6]], 0 -; CHECK-VF8-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], [[TMP7]] -; CHECK-VF8-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP8]] -; CHECK-VF8-NEXT: [[TMP9:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0 -; CHECK-VF8-NEXT: store zeroinitializer, ptr [[TMP9]], align 1 -; CHECK-VF8-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 16 -; CHECK-VF8-NEXT: [[TMP12:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP11]] -; CHECK-VF8-NEXT: store zeroinitializer, ptr [[TMP12]], align 1 -; CHECK-VF8-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-VF8-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 32 -; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]] +; CHECK-VF8-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-VF8-NEXT: [[NEXT_GEP:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP6]] +; CHECK-VF8-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 16 +; CHECK-VF8-NEXT: [[TMP9:%.*]] = add i64 [[TMP8]], 0 +; CHECK-VF8-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], [[TMP9]] +; CHECK-VF8-NEXT: [[NEXT_GEP1:%.*]] = getelementptr i8, ptr [[START]], i64 [[TMP10]] +; CHECK-VF8-NEXT: [[TMP11:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i32 0 +; CHECK-VF8-NEXT: store zeroinitializer, ptr [[TMP11]], align 1 +; CHECK-VF8-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-VF8-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 16 +; CHECK-VF8-NEXT: [[TMP14:%.*]] = getelementptr i8, ptr [[NEXT_GEP]], i64 [[TMP13]] +; CHECK-VF8-NEXT: store zeroinitializer, ptr [[TMP14]], align 1 +; CHECK-VF8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-VF8-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-VF8-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK-VF8: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll index 995950e46f53..061b59f5442c 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-fneg.ll @@ -31,29 +31,29 @@ define void @fneg(ptr nocapture noundef writeonly %d, ptr nocapture noundef read ; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 16 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[WIDE_TRIP_COUNT]], [[TMP7]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[WIDE_TRIP_COUNT]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 16 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP8:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds half, ptr [[S]], i64 [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds half, ptr [[TMP9]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP10]], align 2 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 8 -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds half, ptr [[TMP9]], i64 [[TMP12]] -; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP13]], align 2 -; CHECK-NEXT: [[TMP14:%.*]] = fneg [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP15:%.*]] = fneg [[WIDE_LOAD3]] -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds half, ptr [[D]], i64 [[TMP8]] -; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds half, ptr [[TMP16]], i32 0 -; CHECK-NEXT: store [[TMP14]], ptr [[TMP17]], align 2 -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 8 -; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds half, ptr [[TMP16]], i64 [[TMP19]] -; CHECK-NEXT: store [[TMP15]], ptr [[TMP20]], align 2 -; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 16 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP22]] +; CHECK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds half, ptr [[S]], i64 [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds half, ptr [[TMP11]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP12]], align 2 +; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 8 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds half, ptr [[TMP11]], i64 [[TMP14]] +; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP15]], align 2 +; CHECK-NEXT: [[TMP16:%.*]] = fneg [[WIDE_LOAD]] +; CHECK-NEXT: [[TMP17:%.*]] = fneg [[WIDE_LOAD3]] +; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds half, ptr [[D]], i64 [[TMP10]] +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds half, ptr [[TMP18]], i32 0 +; CHECK-NEXT: store [[TMP16]], ptr [[TMP19]], align 2 +; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 8 +; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds half, ptr [[TMP18]], i64 [[TMP21]] +; CHECK-NEXT: store [[TMP17]], ptr [[TMP22]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; CHECK-NEXT: [[TMP23:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP23]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll index 3a867cf401b3..a9657971d48a 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-gather-scatter.ll @@ -13,18 +13,18 @@ define void @gather_nxv4i32_ind64(ptr noalias nocapture readonly %a, ptr noalias ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP3]], align 8 -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], [[WIDE_LOAD]] -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( [[TMP4]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[C:%.*]], i64 [[INDEX]] -; CHECK-NEXT: store [[WIDE_MASKED_GATHER]], ptr [[TMP5]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP5]], align 8 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], [[WIDE_LOAD]] +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( [[TMP6]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[C:%.*]], i64 [[INDEX]] +; CHECK-NEXT: store [[WIDE_MASKED_GATHER]], ptr [[TMP7]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -80,19 +80,19 @@ define void @scatter_nxv4i32_ind32(ptr noalias nocapture %a, ptr noalias nocaptu ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[C:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP3]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP4]], align 4 -; CHECK-NEXT: [[TMP5:%.*]] = sext [[WIDE_LOAD1]] to -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], [[TMP5]] -; CHECK-NEXT: call void @llvm.masked.scatter.nxv4f32.nxv4p0( [[WIDE_LOAD]], [[TMP6]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds float, ptr [[C:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP5]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load , ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = sext [[WIDE_LOAD1]] to +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], [[TMP7]] +; CHECK-NEXT: call void @llvm.masked.scatter.nxv4f32.nxv4p0( [[WIDE_LOAD]], [[TMP8]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: @@ -147,18 +147,18 @@ define void @scatter_inv_nxv4i32(ptr noalias nocapture %inv, ptr noalias nocaptu ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[INV:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP3]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = icmp ne [[WIDE_LOAD]], zeroinitializer -; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( shufflevector ( insertelement ( poison, i32 3, i64 0), poison, zeroinitializer), [[BROADCAST_SPLAT]], i32 4, [[TMP4]]) -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP5]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = icmp ne [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( shufflevector ( insertelement ( poison, i32 3, i64 0), poison, zeroinitializer), [[BROADCAST_SPLAT]], i32 4, [[TMP6]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: middle.block: @@ -217,19 +217,19 @@ define void @gather_inv_nxv4i32(ptr noalias nocapture %a, ptr noalias nocapture ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[INV:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP3]], align 4 -; CHECK-NEXT: [[TMP4:%.*]] = icmp sgt [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 3, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[BROADCAST_SPLAT]], i32 4, [[TMP4]], poison) -; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[WIDE_MASKED_GATHER]], ptr [[TMP3]], i32 4, [[TMP4]]) -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP6:%.*]] = shl nuw nsw i64 [[TMP5]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP5]], align 4 +; CHECK-NEXT: [[TMP6:%.*]] = icmp sgt [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 3, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[BROADCAST_SPLAT]], i32 4, [[TMP6]], poison) +; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[WIDE_MASKED_GATHER]], ptr [[TMP5]], i32 4, [[TMP6]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] ; CHECK: middle.block: @@ -292,31 +292,31 @@ define void @gather_nxv4i32_ind64_stride2(ptr noalias nocapture %a, ptr noalias ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP2]], -8 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] -; CHECK-NEXT: [[TMP3:%.*]] = call @llvm.experimental.stepvector.nxv4i64() -; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 3 +; CHECK-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv4i64() +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP7]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP3]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[STEP_ADD:%.*]] = add [[VEC_IND]], [[DOTSPLAT]] -; CHECK-NEXT: [[TMP6:%.*]] = shl [[VEC_IND]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP7:%.*]] = shl [[STEP_ADD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], [[TMP6]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[B]], [[TMP7]] -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( [[TMP8]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) -; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( [[TMP9]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: store [[WIDE_MASKED_GATHER]], ptr [[TMP10]], align 4 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = shl nuw nsw i64 [[TMP11]], 2 -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds float, ptr [[TMP10]], i64 [[TMP12]] -; CHECK-NEXT: store [[WIDE_MASKED_GATHER2]], ptr [[TMP13]], align 4 -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 3 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] +; CHECK-NEXT: [[TMP8:%.*]] = shl [[VEC_IND]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP9:%.*]] = shl [[STEP_ADD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[B:%.*]], [[TMP8]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[B]], [[TMP9]] +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( [[TMP10]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) +; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call @llvm.masked.gather.nxv4f32.nxv4p0( [[TMP11]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: store [[WIDE_MASKED_GATHER]], ptr [[TMP12]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP14:%.*]] = shl nuw nsw i64 [[TMP13]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[TMP12]], i64 [[TMP14]] +; CHECK-NEXT: store [[WIDE_MASKED_GATHER2]], ptr [[TMP15]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[STEP_ADD]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll index 01657493e9d4..0fe8fa3f4154 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions-unusual-types.ll @@ -11,7 +11,10 @@ target triple = "aarch64-unknown-linux-gnu" define void @induction_i7(ptr %dst) #0 { ; CHECK-LABEL: @induction_i7( ; CHECK: vector.ph: -; CHECK: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i8() +; CHECK: %ind.end = trunc i64 %n.vec to i7 +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 +; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i8() ; CHECK: [[TMP5:%.*]] = trunc [[TMP4]] to ; CHECK-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer ; CHECK-NEXT: [[TMP7:%.*]] = mul [[TMP6]], shufflevector ( insertelement ( poison, i7 1, i64 0), poison, zeroinitializer) @@ -25,8 +28,6 @@ define void @induction_i7(ptr %dst) #0 { ; CHECK-NEXT: [[EXT:%.+]] = zext [[TMP11]] to ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0 ; CHECK-NEXT: store [[EXT]], ptr [[TMP13]], align 8 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], ; @@ -57,6 +58,9 @@ for.end: ; preds = %for.body define void @induction_i3_zext(ptr %dst) #0 { ; CHECK-LABEL: @induction_i3_zext( ; CHECK: vector.ph: +; CHECK: %ind.end = trunc i64 %n.vec to i3 +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 ; CHECK: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i8() ; CHECK: [[TMP5:%.*]] = trunc [[TMP4]] to ; CHECK-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer @@ -70,8 +74,6 @@ define void @induction_i3_zext(ptr %dst) #0 { ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[DST:%.*]], i64 [[TMP9]] ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0 ; CHECK-NEXT: store [[TMP10]], ptr [[TMP13]], align 8 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], ; diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll index 739010b5b631..965c71c008aa 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inductions.ll @@ -23,26 +23,26 @@ define void @cond_ind64(ptr noalias nocapture %a, ptr noalias nocapture readonly ; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] -; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i64() -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP6]], i64 0 +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 2 +; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv4i64() +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP7]], 2 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP8]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP7:%.*]] = trunc [[VEC_IND]] to -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i32, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP8]], i32 4, [[TMP7]], poison) -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[WIDE_MASKED_LOAD]], ptr [[TMP10]], i32 4, [[TMP7]]) -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = shl i64 [[TMP12]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP9:%.*]] = trunc [[VEC_IND]] to +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP10]], i32 4, [[TMP9]], poison) +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[WIDE_MASKED_LOAD]], ptr [[TMP11]], i32 4, [[TMP9]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] -; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: br i1 [[CMP_N]], label [[EXIT:%.*]], label [[SCALAR_PH]] @@ -56,14 +56,14 @@ define void @cond_ind64(ptr noalias nocapture %a, ptr noalias nocapture readonly ; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]] ; CHECK: if.then: ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[I_08]] -; CHECK-NEXT: [[TMP15:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 ; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[I_08]] -; CHECK-NEXT: store i32 [[TMP15]], ptr [[ARRAYIDX1]], align 4 +; CHECK-NEXT: store i32 [[TMP13]], ptr [[ARRAYIDX1]], align 4 ; CHECK-NEXT: br label [[FOR_INC]] ; CHECK: for.inc: ; CHECK-NEXT: [[INC]] = add nuw nsw i64 [[I_08]], 1 ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INC]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[EXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: exit: ; CHECK-NEXT: ret void ; diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll index 0b5058cff8d5..f600a0d5877d 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-accesses.ll @@ -26,6 +26,8 @@ define void @test_array_load2_store2(i32 %C, i32 %D) #1 { ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[C:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[D:%.*]], i64 0 @@ -46,8 +48,6 @@ define void @test_array_load2_store2(i32 %C, i32 %D) #1 { ; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[TMP6]], i64 -1 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv8i32( [[TMP4]], [[TMP5]]) ; CHECK-NEXT: store [[INTERLEAVED_VEC]], ptr [[TMP7]], align 4 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512 ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -106,6 +106,8 @@ define void @test_array_load2_i16_store2(i32 %C, i32 %D) #1 { ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[TMP0:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl [[TMP0]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() @@ -134,8 +136,6 @@ define void @test_array_load2_i16_store2(i32 %C, i32 %D) #1 { ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i64 -1 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv8i32( [[TMP8]], [[TMP10]]) ; CHECK-NEXT: store [[INTERLEAVED_VEC]], ptr [[TMP13]], align 4 -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512 @@ -197,6 +197,8 @@ define void @test_array_load2_store2_i16(i32 noundef %C, i32 noundef %D) #1 { ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[TMP0:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl [[TMP0]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.vscale.i64() @@ -226,8 +228,6 @@ define void @test_array_load2_store2_i16(i32 noundef %C, i32 noundef %D) #1 { ; CHECK-NEXT: [[TMP12:%.*]] = trunc [[TMP11]] to ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds [1024 x i16], ptr @CD_i16, i64 0, [[TMP7]] ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i16.nxv4p0( [[TMP12]], [[TMP13]], i32 2, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512 @@ -277,6 +277,8 @@ define i32 @test_struct_load6(%struct.ST6* %S) #1 { ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 2 ; CHECK-NEXT: [[TMP0:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP2:%.*]] = shl nuw nsw i64 [[TMP1]], 2 @@ -305,8 +307,6 @@ define i32 @test_struct_load6(%struct.ST6* %S) #1 { ; CHECK-NEXT: [[TMP12:%.*]] = add [[TMP11]], [[WIDE_MASKED_GATHER4]] ; CHECK-NEXT: [[TMP13:%.*]] = add [[TMP12]], [[WIDE_MASKED_GATHER5]] ; CHECK-NEXT: [[TMP14]] = sub [[TMP10]], [[TMP13]] -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 @@ -381,6 +381,8 @@ define void @test_reversed_load2_store2(%struct.ST2* noalias nocapture readonly ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP19:%.*]] = shl nuw nsw i64 [[TMP18]], 2 ; CHECK-NEXT: [[TMP0:%.*]] = call @llvm.experimental.stepvector.nxv4i32() ; CHECK-NEXT: [[INDUCTION:%.*]] = sub shufflevector ( insertelement ( poison, i32 1023, i64 0), poison, zeroinitializer), [[TMP0]] ; CHECK-NEXT: [[TMP1:%.*]] = call i32 @llvm.vscale.i32() @@ -416,8 +418,6 @@ define void @test_reversed_load2_store2(%struct.ST2* noalias nocapture readonly ; CHECK-NEXT: [[REVERSE3:%.*]] = call @llvm.experimental.vector.reverse.nxv4i32( [[TMP11]]) ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv8i32( [[REVERSE2]], [[REVERSE3]]) ; CHECK-NEXT: store [[INTERLEAVED_VEC]], ptr [[TMP17]], align 4 -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = shl nuw nsw i64 [[TMP18]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 @@ -475,6 +475,8 @@ define void @even_load_static_tc(i32* noalias nocapture readonly %A, i32* noalia ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2 ; CHECK-NEXT: [[N_VEC:%.*]] = sub nuw nsw i64 512, [[TMP1]] ; CHECK-NEXT: [[IND_END:%.*]] = shl nuw nsw i64 [[N_VEC]], 1 +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -487,8 +489,6 @@ define void @even_load_static_tc(i32* noalias nocapture readonly %A, i32* noalia ; CHECK-NEXT: [[TMP5:%.*]] = and i64 [[INDEX]], 9223372036854775804 ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP5]] ; CHECK-NEXT: store [[TMP4]], ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = shl nuw nsw i64 [[TMP7]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -561,6 +561,8 @@ define void @even_load_dynamic_tc(i32* noalias nocapture readonly %A, i32* noali ; CHECK-NEXT: [[TMP9:%.*]] = select i1 [[TMP8]], i64 [[TMP6]], i64 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP4]], [[TMP9]] ; CHECK-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1 +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -573,8 +575,6 @@ define void @even_load_dynamic_tc(i32* noalias nocapture readonly %A, i32* noali ; CHECK-NEXT: [[TMP13:%.*]] = and i64 [[INDEX]], 9223372036854775804 ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP13]] ; CHECK-NEXT: store [[TMP12]], ptr [[TMP14]], align 4 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] @@ -640,6 +640,8 @@ define void @load_gap_reverse(%pair* noalias nocapture readonly %P1, %pair* noal ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 ; CHECK-NEXT: [[TMP0:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[INDUCTION:%.*]] = sub shufflevector ( insertelement ( poison, i64 1023, i64 0), poison, zeroinitializer), [[TMP0]] ; CHECK-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() @@ -659,8 +661,6 @@ define void @load_gap_reverse(%pair* noalias nocapture readonly %P1, %pair* noal ; CHECK-NEXT: [[TMP5:%.*]] = sub nsw [[WIDE_MASKED_GATHER]], [[VEC_IND]] ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i64.nxv4p0( [[TMP2]], [[TMP3]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i64.nxv4p0( [[TMP5]], [[TMP4]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 @@ -709,6 +709,8 @@ define void @mixed_load2_store2(i32* noalias nocapture readonly %A, i32* noalias ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -728,8 +730,6 @@ define void @mixed_load2_store2(i32* noalias nocapture readonly %A, i32* noalias ; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[TMP8]], i64 -1 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv8i32( [[TMP4]], [[TMP7]]) ; CHECK-NEXT: store [[INTERLEAVED_VEC]], ptr [[TMP9]], align 4 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = shl nuw nsw i64 [[TMP10]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], 512 ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] @@ -802,6 +802,8 @@ define void @int_float_struct(%struct.IntFloat* nocapture readonly %p) #0 { ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -815,8 +817,6 @@ define void @int_float_struct(%struct.IntFloat* nocapture readonly %p) #0 { ; CHECK-NEXT: [[TMP3:%.*]] = bitcast [[TMP2]] to ; CHECK-NEXT: [[TMP4]] = add [[TMP1]], [[VEC_PHI1]] ; CHECK-NEXT: [[TMP5]] = fadd fast [[VEC_PHI]], [[TMP3]] -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP20:![0-9]+]] @@ -892,6 +892,8 @@ define void @PR27626_0(%pair.i32 *%p, i32 %z, i64 %n) #1 { ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]] +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2 @@ -911,8 +913,6 @@ define void @PR27626_0(%pair.i32 *%p, i32 %z, i64 %n) #1 { ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { , } @llvm.experimental.vector.deinterleave2.nxv8i32( [[WIDE_VEC]]) ; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { , } [[STRIDED_VEC]], 0 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP13]], [[TMP11]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -979,6 +979,8 @@ define i32 @PR27626_1(%pair.i32 *%p, i64 %n) #1 { ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]] +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2 @@ -1000,8 +1002,6 @@ define i32 @PR27626_1(%pair.i32 *%p, i64 %n) #1 { ; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { , } @llvm.experimental.vector.deinterleave2.nxv8i32( [[WIDE_VEC1]]) ; CHECK-NEXT: [[TMP14:%.*]] = extractvalue { , } [[STRIDED_VEC2]], 0 ; CHECK-NEXT: [[TMP15]] = add [[TMP14]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1074,6 +1074,8 @@ define void @PR27626_2(%pair.i32 *%p, i64 %n, i32 %z) #1 { ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]] +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2 @@ -1093,8 +1095,6 @@ define void @PR27626_2(%pair.i32 *%p, i64 %n, i32 %z) #1 { ; CHECK-NEXT: [[STRIDED_VEC:%.*]] = call { , } @llvm.experimental.vector.deinterleave2.nxv8i32( [[WIDE_VEC]]) ; CHECK-NEXT: [[TMP13:%.*]] = extractvalue { , } [[STRIDED_VEC]], 0 ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP13]], [[TMP12]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1164,6 +1164,8 @@ define i32 @PR27626_3(%pair.i32 *%p, i64 %n, i32 %z) #1 { ; CHECK-NEXT: [[TMP5:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP6:%.*]] = select i1 [[TMP5]], i64 [[TMP3]], i64 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[SMAX]], [[TMP6]] +; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP18:%.*]] = shl nuw nsw i64 [[TMP17]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP9:%.*]] = shl nuw nsw i64 [[TMP8]], 2 @@ -1186,8 +1188,6 @@ define i32 @PR27626_3(%pair.i32 *%p, i64 %n, i32 %z) #1 { ; CHECK-NEXT: [[STRIDED_VEC2:%.*]] = call { , } @llvm.experimental.vector.deinterleave2.nxv8i32( [[WIDE_VEC1]]) ; CHECK-NEXT: [[TMP15:%.*]] = extractvalue { , } [[STRIDED_VEC2]], 0 ; CHECK-NEXT: [[TMP16]] = add [[TMP15]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP18:%.*]] = shl nuw nsw i64 [[TMP17]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1267,6 +1267,8 @@ define void @PR27626_4(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) #1 { ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP5]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], [[DOTNEG]] ; CHECK-NEXT: [[IND_END:%.*]] = shl nuw i64 [[N_VEC]], 1 +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP7:%.*]] = shl [[TMP6]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() @@ -1291,8 +1293,6 @@ define void @PR27626_4(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) #1 { ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i64 -1 ; CHECK-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv8i32( [[BROADCAST_SPLAT2]], [[BROADCAST_SPLAT4]]) ; CHECK-NEXT: store [[INTERLEAVED_VEC]], ptr [[TMP13]], align 4 -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl nuw nsw i64 [[TMP14]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1363,6 +1363,8 @@ define void @PR27626_5(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) #1 { ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP2]], [[DOTNEG]] ; CHECK-NEXT: [[TMP6:%.*]] = shl nuw i64 [[N_VEC]], 1 ; CHECK-NEXT: [[IND_END:%.*]] = or disjoint i64 [[TMP6]], 3 +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 2 ; CHECK-NEXT: [[TMP7:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP8:%.*]] = shl [[TMP7]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[INDUCTION:%.*]] = add [[TMP8]], shufflevector ( insertelement ( poison, i64 3, i64 0), poison, zeroinitializer) @@ -1388,8 +1390,6 @@ define void @PR27626_5(i32 *%a, i32 %x, i32 %y, i32 %z, i64 %n) #1 { ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[BROADCAST_SPLAT]], [[TMP14]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[BROADCAST_SPLAT2]], [[TMP15]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[BROADCAST_SPLAT4]], [[TMP13]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP17:%.*]] = shl nuw nsw i64 [[TMP16]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1473,6 +1473,8 @@ define void @PR34743(i16* %a, i32* %b, i64 %n) #1 { ; CHECK-NEXT: [[DOTNEG:%.*]] = mul nsw i64 [[TMP8]], -4 ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[TMP1]], [[DOTNEG]] ; CHECK-NEXT: [[IND_END:%.*]] = shl i64 [[N_VEC]], 1 +; CHECK-NEXT: [[TMP27:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP27]], 2 ; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() ; CHECK-NEXT: [[TMP10:%.*]] = shl nuw nsw i32 [[TMP9]], 2 ; CHECK-NEXT: [[TMP11:%.*]] = add nsw i32 [[TMP10]], -1 @@ -1501,9 +1503,7 @@ define void @PR34743(i16* %a, i32* %b, i64 %n) #1 { ; CHECK-NEXT: [[TMP24:%.*]] = mul nsw [[TMP22]], [[TMP19]] ; CHECK-NEXT: [[TMP25:%.*]] = mul nsw [[TMP24]], [[TMP23]] ; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] -; CHECK-NEXT: store [[TMP25]], ptr [[TMP26]], align 4, !alias.scope [[META37:![0-9]+]], !noalias [[META34]] -; CHECK-NEXT: [[TMP27:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP28:%.*]] = shl nuw nsw i64 [[TMP27]], 2 +; CHECK-NEXT: store [[TMP25]], ptr [[TMP26]], align 4, !alias.scope [[META37:![0-9]+]], !noalias [[META34]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP28]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll index 56e2fe6e15b7..1b5df2c1bfb8 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-interleaved-masked-accesses.ll @@ -35,6 +35,8 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no ; SCALAR_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]] ; SCALAR_TAIL_FOLDING-NEXT: [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]] +; SCALAR_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = call i32 @llvm.vscale.i32() +; SCALAR_TAIL_FOLDING-NEXT: [[TMP20:%.*]] = shl i32 [[TMP19]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv16i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = call i32 @llvm.vscale.i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = shl i32 [[TMP5]], 4 @@ -64,8 +66,6 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no ; SCALAR_TAIL_FOLDING-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv32i8( [[TMP14]], [[TMP15]]) ; SCALAR_TAIL_FOLDING-NEXT: [[INTERLEAVED_MASK1:%.*]] = call @llvm.experimental.vector.interleave2.nxv32i1( [[TMP7]], [[TMP7]]) ; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.store.nxv32i8.p0( [[INTERLEAVED_VEC]], ptr [[TMP18]], i32 1, [[INTERLEAVED_MASK1]]) -; SCALAR_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = call i32 @llvm.vscale.i32() -; SCALAR_TAIL_FOLDING-NEXT: [[TMP20:%.*]] = shl i32 [[TMP19]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP20]] ; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP21:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] @@ -111,6 +111,8 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no ; PREDICATED_TAIL_FOLDING-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; PREDICATED_TAIL_FOLDING: vector.ph: ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = call i32 @llvm.vscale.i32() +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP20:%.*]] = shl i32 [[TMP19]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP1]]) @@ -146,8 +148,6 @@ define dso_local void @masked_strided1(ptr noalias nocapture readonly %p, ptr no ; PREDICATED_TAIL_FOLDING-NEXT: [[INTERLEAVED_VEC:%.*]] = call @llvm.experimental.vector.interleave2.nxv32i8( [[TMP14]], [[TMP15]]) ; PREDICATED_TAIL_FOLDING-NEXT: [[INTERLEAVED_MASK1:%.*]] = call @llvm.experimental.vector.interleave2.nxv32i1( [[TMP10]], [[TMP10]]) ; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.store.nxv32i8.p0( [[INTERLEAVED_VEC]], ptr [[TMP18]], i32 1, [[INTERLEAVED_MASK1]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP19:%.*]] = call i32 @llvm.vscale.i32() -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP20:%.*]] = shl i32 [[TMP19]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP20]] ; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP2]]) ; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] @@ -227,6 +227,8 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no ; SCALAR_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]] ; SCALAR_TAIL_FOLDING-NEXT: [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]] +; SCALAR_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32() +; SCALAR_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = shl i32 [[TMP14]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv16i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = call i32 @llvm.vscale.i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = shl i32 [[TMP5]], 4 @@ -247,8 +249,6 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no ; SCALAR_TAIL_FOLDING-NEXT: [[TMP12:%.*]] = zext nneg [[TMP11]] to ; SCALAR_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP12]] ; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( shufflevector ( insertelement ( poison, i8 2, i64 0), poison, zeroinitializer), [[TMP13]], i32 1, [[TMP10]]) -; SCALAR_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32() -; SCALAR_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = shl i32 [[TMP14]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP15]] ; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] @@ -286,6 +286,8 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no ; PREDICATED_TAIL_FOLDING-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; PREDICATED_TAIL_FOLDING: vector.ph: ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD]] to i32 +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32() +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = shl i32 [[TMP14]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP1]]) @@ -312,8 +314,6 @@ define dso_local void @masked_strided2(ptr noalias nocapture readnone %p, ptr no ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP11]] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP9]], zeroinitializer ; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( shufflevector ( insertelement ( poison, i8 2, i64 0), poison, zeroinitializer), [[TMP12]], i32 1, [[TMP13]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32() -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = shl i32 [[TMP14]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP15]] ; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP2]]) ; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] @@ -389,6 +389,8 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no ; SCALAR_TAIL_FOLDING-NEXT: [[TMP3:%.*]] = shl i32 [[TMP2]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[N_MOD_VF:%.*]] = urem i32 1024, [[TMP3]] ; SCALAR_TAIL_FOLDING-NEXT: [[N_VEC:%.*]] = sub nuw nsw i32 1024, [[N_MOD_VF]] +; SCALAR_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = call i32 @llvm.vscale.i32() +; SCALAR_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = shl i32 [[TMP15]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv16i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP5:%.*]] = call i32 @llvm.vscale.i32() ; SCALAR_TAIL_FOLDING-NEXT: [[TMP6:%.*]] = shl i32 [[TMP5]], 4 @@ -412,8 +414,6 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no ; SCALAR_TAIL_FOLDING-NEXT: [[TMP13:%.*]] = zext nneg [[TMP12]] to ; SCALAR_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP13]] ; SCALAR_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( shufflevector ( insertelement ( poison, i8 2, i64 0), poison, zeroinitializer), [[TMP14]], i32 1, [[TMP11]]) -; SCALAR_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = call i32 @llvm.vscale.i32() -; SCALAR_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = shl i32 [[TMP15]], 4 ; SCALAR_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP16]] ; SCALAR_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; SCALAR_TAIL_FOLDING-NEXT: [[TMP17:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] @@ -457,6 +457,8 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no ; PREDICATED_TAIL_FOLDING: vector.ph: ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV3:%.*]] = zext i8 [[GUARD2]] to i32 ; PREDICATED_TAIL_FOLDING-NEXT: [[CONV:%.*]] = zext i8 [[GUARD1]] to i32 +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = call i32 @llvm.vscale.i32() +; PREDICATED_TAIL_FOLDING-NEXT: [[TMP17:%.*]] = shl i32 [[TMP16]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP0:%.*]] = call i32 @llvm.vscale.i32() ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP2:%.*]] = call i32 @llvm.usub.sat.i32(i32 1024, i32 [[TMP1]]) @@ -487,8 +489,6 @@ define dso_local void @masked_strided3(ptr noalias nocapture readnone %p, ptr no ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP14:%.*]] = getelementptr inbounds i8, ptr [[Q]], [[TMP13]] ; PREDICATED_TAIL_FOLDING-NEXT: [[TMP15:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP11]], zeroinitializer ; PREDICATED_TAIL_FOLDING-NEXT: call void @llvm.masked.scatter.nxv16i8.nxv16p0( shufflevector ( insertelement ( poison, i8 2, i64 0), poison, zeroinitializer), [[TMP14]], i32 1, [[TMP15]]) -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP16:%.*]] = call i32 @llvm.vscale.i32() -; PREDICATED_TAIL_FOLDING-NEXT: [[TMP17:%.*]] = shl i32 [[TMP16]], 4 ; PREDICATED_TAIL_FOLDING-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP17]] ; PREDICATED_TAIL_FOLDING-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv16i1.i32(i32 [[INDEX]], i32 [[TMP2]]) ; PREDICATED_TAIL_FOLDING-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll index 6a562e8b92bb..08da1dedae23 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-inv-store.ll @@ -15,21 +15,21 @@ define void @inv_store_i16(ptr noalias %dst, ptr noalias readonly %src, i64 %N) ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[SRC:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 2 -; CHECK-NEXT: [[TMP7:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP8:%.*]] = mul i32 [[TMP7]], 4 -; CHECK-NEXT: [[TMP9:%.*]] = sub i32 [[TMP8]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = extractelement [[WIDE_LOAD]], i32 [[TMP9]] -; CHECK-NEXT: store i16 [[TMP10]], ptr [[DST:%.*]], align 2 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[SRC:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 2 +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 4 +; CHECK-NEXT: [[TMP11:%.*]] = sub i32 [[TMP10]], 1 +; CHECK-NEXT: [[TMP12:%.*]] = extractelement [[WIDE_LOAD]], i32 [[TMP11]] +; CHECK-NEXT: store i16 [[TMP12]], ptr [[DST:%.*]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -45,7 +45,7 @@ define void @inv_store_i16(ptr noalias %dst, ptr noalias readonly %src, i64 %N) ; CHECK-NEXT: store i16 [[LD]], ptr [[DST]], align 2 ; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_INC24]], label [[FOR_BODY14]], !llvm.loop [[LOOP3:![0-9]+]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_INC24]], label [[FOR_BODY14]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: for.inc24: ; CHECK-NEXT: ret void ; @@ -78,20 +78,20 @@ define void @cond_inv_store_i32(ptr noalias %dst, ptr noalias readonly %src, i64 ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[DST:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = icmp sgt [[WIDE_LOAD]], zeroinitializer -; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[WIDE_LOAD]], [[BROADCAST_SPLAT]], i32 4, [[TMP7]]) -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[SRC:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 4 +; CHECK-NEXT: [[TMP9:%.*]] = icmp sgt [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[WIDE_LOAD]], [[BROADCAST_SPLAT]], i32 4, [[TMP9]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll index b378603bbc52..d12a1036ed53 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-live-out-pointer-induction.ll @@ -23,6 +23,8 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) { ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START_1:%.*]], i64 [[TMP8]] ; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[N_VEC]], 8 ; CHECK-NEXT: [[IND_END3:%.*]] = getelementptr i8, ptr [[START_2]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP34:%.*]] = mul i64 [[TMP33]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START_1]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] @@ -60,8 +62,6 @@ define ptr @test(ptr %start.1, ptr %start.2, ptr %end) { ; CHECK-NEXT: [[TMP31:%.*]] = mul i64 [[TMP30]], 2 ; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i64, ptr [[NEXT_GEP]], i64 [[TMP31]] ; CHECK-NEXT: store zeroinitializer, ptr [[TMP32]], align 8 -; CHECK-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP34:%.*]] = mul i64 [[TMP33]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP34]] ; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP13]] ; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll index 997aca89b825..602ccb678c96 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-low-trip-count.ll @@ -4,14 +4,18 @@ target triple = "aarch64-unknown-linux-gnu" define void @trip7_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapture noundef readonly %src) #0 { ; CHECK-LABEL: @trip7_i64( +; CHECK: = call i64 @llvm.vscale.i64() +; CHECK-NEXT: = mul i64 +; CHECK: = call i64 @llvm.vscale.i64() +; CHECK-NEXT: = mul i64 +; CHECK: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[VF:%.*]] = mul i64 [[VSCALE]], 2 ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] ; CHECK: [[ACTIVE_LANE_MASK:%.*]] = phi [ {{%.*}}, %vector.ph ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], %vector.body ] ; CHECK: {{%.*}} = call @llvm.masked.load.nxv2i64.p0(ptr {{%.*}}, i32 8, [[ACTIVE_LANE_MASK]], poison) ; CHECK: {{%.*}} = call @llvm.masked.load.nxv2i64.p0(ptr {{%.*}}, i32 8, [[ACTIVE_LANE_MASK]], poison) ; CHECK: call void @llvm.masked.store.nxv2i64.p0( {{%.*}}, ptr {{%.*}}, i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK: [[VSCALE:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[VF:%.*]] = mul i64 [[VSCALE]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[VF]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 7) ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NOT:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll index 79bda0a4a5d8..006171f3b068 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-multiexit.ll @@ -33,34 +33,34 @@ define void @multiple_exits_unique_exit_block(ptr %A, ptr %B, i32 %N) #0 { ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP10]] +; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP12:%.*]] = mul i32 [[TMP11]], 8 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[INDEX]], 0 -; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP13:%.*]] = mul i32 [[TMP12]], 4 -; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 0 -; CHECK-NEXT: [[TMP15:%.*]] = mul i32 [[TMP14]], 1 -; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[INDEX]], [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP11]] -; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP16]] -; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP19]], align 4 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i64 [[TMP21]] -; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP22]], align 4 -; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP11]] -; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP16]] -; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i32 0 -; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP25]], align 4 -; CHECK-NEXT: [[TMP26:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP27:%.*]] = mul i64 [[TMP26]], 4 -; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i64 [[TMP27]] -; CHECK-NEXT: store [[WIDE_LOAD3]], ptr [[TMP28]], align 4 -; CHECK-NEXT: [[TMP29:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP30:%.*]] = mul i32 [[TMP29]], 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP30]] +; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 0 +; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP15:%.*]] = mul i32 [[TMP14]], 4 +; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], 0 +; CHECK-NEXT: [[TMP17:%.*]] = mul i32 [[TMP16]], 1 +; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[INDEX]], [[TMP17]] +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP13]] +; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP18]] +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4 +; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 [[TMP23]] +; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP24]], align 4 +; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP13]] +; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP18]] +; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i32 0 +; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP27]], align 4 +; CHECK-NEXT: [[TMP28:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP29:%.*]] = mul i64 [[TMP28]], 4 +; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP29]] +; CHECK-NEXT: store [[WIDE_LOAD3]], ptr [[TMP30]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP12]] ; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -131,34 +131,34 @@ define i32 @multiple_exits_multiple_exit_blocks(ptr %A, ptr %B, i32 %N) #0 { ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i32 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP10:%.*]] = select i1 [[TMP9]], i32 [[TMP8]], i32 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[TMP0]], [[TMP10]] +; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP12:%.*]] = mul i32 [[TMP11]], 8 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP11:%.*]] = add i32 [[INDEX]], 0 -; CHECK-NEXT: [[TMP12:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP13:%.*]] = mul i32 [[TMP12]], 4 -; CHECK-NEXT: [[TMP14:%.*]] = add i32 [[TMP13]], 0 -; CHECK-NEXT: [[TMP15:%.*]] = mul i32 [[TMP14]], 1 -; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[INDEX]], [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP11]] -; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP16]] -; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP19]], align 4 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i64 [[TMP21]] -; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP22]], align 4 -; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP11]] -; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP16]] -; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i32 0 -; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP25]], align 4 -; CHECK-NEXT: [[TMP26:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP27:%.*]] = mul i64 [[TMP26]], 4 -; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i32, ptr [[TMP23]], i64 [[TMP27]] -; CHECK-NEXT: store [[WIDE_LOAD3]], ptr [[TMP28]], align 4 -; CHECK-NEXT: [[TMP29:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP30:%.*]] = mul i32 [[TMP29]], 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP30]] +; CHECK-NEXT: [[TMP13:%.*]] = add i32 [[INDEX]], 0 +; CHECK-NEXT: [[TMP14:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP15:%.*]] = mul i32 [[TMP14]], 4 +; CHECK-NEXT: [[TMP16:%.*]] = add i32 [[TMP15]], 0 +; CHECK-NEXT: [[TMP17:%.*]] = mul i32 [[TMP16]], 1 +; CHECK-NEXT: [[TMP18:%.*]] = add i32 [[INDEX]], [[TMP17]] +; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP13]] +; CHECK-NEXT: [[TMP20:%.*]] = getelementptr inbounds i32, ptr [[A]], i32 [[TMP18]] +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP22:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP23:%.*]] = mul i64 [[TMP22]], 4 +; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP19]], i64 [[TMP23]] +; CHECK-NEXT: [[WIDE_LOAD3:%.*]] = load , ptr [[TMP24]], align 4 +; CHECK-NEXT: [[TMP25:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP13]] +; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[B]], i32 [[TMP18]] +; CHECK-NEXT: [[TMP27:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i32 0 +; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP27]], align 4 +; CHECK-NEXT: [[TMP28:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP29:%.*]] = mul i64 [[TMP28]], 4 +; CHECK-NEXT: [[TMP30:%.*]] = getelementptr inbounds i32, ptr [[TMP25]], i64 [[TMP29]] +; CHECK-NEXT: store [[WIDE_LOAD3]], ptr [[TMP30]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP12]] ; CHECK-NEXT: [[TMP31:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP31]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll index 166d77148244..1d5c60bd7374 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-runtime-check-size-based-threshold.ll @@ -47,6 +47,8 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr ; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP16]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP49:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP50:%.*]] = mul i64 [[TMP49]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -90,8 +92,6 @@ define void @min_trip_count_due_to_runtime_checks_1(ptr %dst.1, ptr %dst.2, ptr ; CHECK-NEXT: [[TMP47:%.*]] = mul i64 [[TMP46]], 2 ; CHECK-NEXT: [[TMP48:%.*]] = getelementptr i64, ptr [[TMP39]], i64 [[TMP47]] ; CHECK-NEXT: store [[TMP36]], ptr [[TMP48]], align 8 -; CHECK-NEXT: [[TMP49:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP50:%.*]] = mul i64 [[TMP49]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP50]] ; CHECK-NEXT: [[TMP51:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP51]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll index 2ca5fbebed87..3aac54114c51 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-forced.ll @@ -10,6 +10,7 @@ target triple = "aarch64-unknown-linux-gnu" ; VPLANS-LABEL: Checking a loop in 'simple_memset' ; VPLANS: VPlan 'Initial VPlan for VF={vscale x 1,vscale x 2,vscale x 4},UF>=1' { +; VPLANS-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; VPLANS-NEXT: vp<[[TC:%[0-9]+]]> = original trip-count ; VPLANS-EMPTY: ; VPLANS-NEXT: ph: @@ -29,7 +30,7 @@ target triple = "aarch64-unknown-linux-gnu" ; VPLANS-NEXT: vp<[[STEP:%[0-9]+]]> = SCALAR-STEPS vp<[[INDV]]>, ir<1> ; VPLANS-NEXT: CLONE ir<%gep> = getelementptr ir<%ptr>, vp<[[STEP]]> ; VPLANS-NEXT: WIDEN store ir<%gep>, ir<%val>, vp<[[LANEMASK_PHI]]> -; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%[0-9]+]]> = VF * UF + vp<[[INDV]]> +; VPLANS-NEXT: EMIT vp<[[INDV_UPDATE:%[0-9]+]]> = add vp<[[INDV]]>, vp<[[VFxUF]]> ; VPLANS-NEXT: EMIT vp<[[INC:%[0-9]+]]> = VF * Part + vp<[[INDV]]> ; VPLANS-NEXT: EMIT vp<[[LANEMASK_LOOP]]> = active lane mask vp<[[INC]]>, vp<[[NEWTC]]> ; VPLANS-NEXT: EMIT vp<[[NOT:%[0-9]+]]> = not vp<[[LANEMASK_LOOP]]> @@ -51,6 +52,8 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -67,8 +70,6 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP10]] ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i32 0 ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP14]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP15:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll index 275a26bae1f2..568046ca29f5 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-optsize.ll @@ -16,24 +16,24 @@ define void @trip1025_i64(ptr noalias nocapture noundef %dst, ptr noalias nocapt ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = phi [ [[ACTIVE_LANE_MASK_ENTRY]], [[VECTOR_PH]] ], [ [[ACTIVE_LANE_MASK_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = shl nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[DST:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP9]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP10]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP11:%.*]] = add nsw [[WIDE_MASKED_LOAD1]], [[TMP8]] -; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP11]], ptr [[TMP10]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[SRC:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP10:%.*]] = shl nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[DST:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP12]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP13:%.*]] = add nsw [[WIDE_MASKED_LOAD1]], [[TMP10]] +; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP13]], ptr [[TMP12]], i32 8, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; CHECK-NEXT: [[TMP14:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP15:%.*]] = extractelement [[TMP14]], i32 0 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-overflow-checks.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-overflow-checks.ll index c0e736399f29..df107847e3e3 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-overflow-checks.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-overflow-checks.ll @@ -15,6 +15,8 @@ define void @cannot_overflow_i32_induction_var(ptr noalias %dst, ptr readonly %s ; CHECK-NEXT: [[WIDE_TRIP_COUNT:%.*]] = zext i32 [[N]] to i64 ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[WIDE_TRIP_COUNT]]) ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: @@ -25,8 +27,6 @@ define void @cannot_overflow_i32_induction_var(ptr noalias %dst, ptr readonly %s ; CHECK-NEXT: [[TMP1:%.*]] = add nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i32 42, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP1]], ptr [[TMP2]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP3:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP4:%.*]] = shl nuw nsw i64 [[TMP3]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP4]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 [[WIDE_TRIP_COUNT]]) ; CHECK-NEXT: [[TMP5:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 @@ -76,6 +76,8 @@ define void @can_overflow_i64_induction_var(ptr noalias %dst, ptr readonly %src, ; CHECK: for.body.preheader: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl nuw nsw i64 [[TMP0]], 2 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP1]]) @@ -89,8 +91,6 @@ define void @can_overflow_i64_induction_var(ptr noalias %dst, ptr readonly %src, ; CHECK-NEXT: [[TMP4:%.*]] = add nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i32 42, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP4]], ptr [[TMP5]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP2]]) ; CHECK-NEXT: [[TMP8:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll index 7e0f18b9efae..70e50992b438 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-reductions.ll @@ -20,6 +20,8 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -37,8 +39,6 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP13:%.*]] = add [[VEC_PHI]], [[WIDE_MASKED_LOAD]] ; CHECK-NEXT: [[TMP14]] = select [[ACTIVE_LANE_MASK]], [[TMP13]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP17:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -77,6 +77,8 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 { ; CHECK-IN-LOOP-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-IN-LOOP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-IN-LOOP-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-IN-LOOP-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-IN-LOOP-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 ; CHECK-IN-LOOP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-IN-LOOP-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-IN-LOOP-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -95,8 +97,6 @@ define i32 @add_reduction_i32(ptr %ptr, i64 %n) #0 { ; CHECK-IN-LOOP-NEXT: [[TMP13:%.*]] = select [[ACTIVE_LANE_MASK]], [[WIDE_MASKED_LOAD]], zeroinitializer ; CHECK-IN-LOOP-NEXT: [[TMP14:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP13]]) ; CHECK-IN-LOOP-NEXT: [[TMP15]] = add i32 [[TMP14]], [[VEC_PHI]] -; CHECK-IN-LOOP-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-IN-LOOP-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 ; CHECK-IN-LOOP-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP17]] ; CHECK-IN-LOOP-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-IN-LOOP-NEXT: [[TMP18:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -152,6 +152,8 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -169,8 +171,6 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP13:%.*]] = select [[ACTIVE_LANE_MASK]], [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP13]]) -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP17:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -208,6 +208,8 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 { ; CHECK-IN-LOOP-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-IN-LOOP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-IN-LOOP-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-IN-LOOP-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-IN-LOOP-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-IN-LOOP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-IN-LOOP-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-IN-LOOP-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -225,8 +227,6 @@ define float @add_reduction_f32(ptr %ptr, i64 %n) #0 { ; CHECK-IN-LOOP-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-IN-LOOP-NEXT: [[TMP13:%.*]] = select [[ACTIVE_LANE_MASK]], [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, float -0.000000e+00, i64 0), poison, zeroinitializer) ; CHECK-IN-LOOP-NEXT: [[TMP14]] = call float @llvm.vector.reduce.fadd.nxv4f32(float [[VEC_PHI]], [[TMP13]]) -; CHECK-IN-LOOP-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-IN-LOOP-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-IN-LOOP-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] ; CHECK-IN-LOOP-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-IN-LOOP-NEXT: [[TMP17:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -281,6 +281,8 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -306,8 +308,6 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 { ; CHECK-NEXT: [[TMP19:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP18]], zeroinitializer ; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP15]], [[TMP17]], [[VEC_PHI]] ; CHECK-NEXT: [[TMP20]] = select [[ACTIVE_LANE_MASK]], [[PREDPHI]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP22]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP23:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -353,6 +353,8 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 { ; CHECK-IN-LOOP-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] ; CHECK-IN-LOOP-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-IN-LOOP-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-IN-LOOP-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-IN-LOOP-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; CHECK-IN-LOOP-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-IN-LOOP-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-IN-LOOP-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -376,8 +378,6 @@ define i32 @cond_xor_reduction(ptr noalias %a, ptr noalias %cond, i64 %N) #0 { ; CHECK-IN-LOOP-NEXT: [[TMP17:%.*]] = select [[TMP15]], [[WIDE_MASKED_LOAD1]], zeroinitializer ; CHECK-IN-LOOP-NEXT: [[TMP18:%.*]] = call i32 @llvm.vector.reduce.xor.nxv4i32( [[TMP17]]) ; CHECK-IN-LOOP-NEXT: [[TMP19]] = xor i32 [[TMP18]], [[VEC_PHI]] -; CHECK-IN-LOOP-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-IN-LOOP-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; CHECK-IN-LOOP-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP21]] ; CHECK-IN-LOOP-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]]) ; CHECK-IN-LOOP-NEXT: [[TMP22:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll index caeb37bce2f1..4f73d7c1a773 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding-unroll.ll @@ -18,6 +18,8 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP61:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP62:%.*]] = mul i64 [[TMP61]], 16 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 16 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -94,8 +96,6 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP59:%.*]] = mul i64 [[TMP58]], 12 ; CHECK-NEXT: [[TMP60:%.*]] = getelementptr i32, ptr [[TMP47]], i64 [[TMP59]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP60]], i32 4, [[ACTIVE_LANE_MASK9]]) -; CHECK-NEXT: [[TMP61:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP62:%.*]] = mul i64 [[TMP61]], 16 ; CHECK-NEXT: [[INDEX_NEXT10]] = add i64 [[INDEX6]], [[TMP62]] ; CHECK-NEXT: [[TMP63:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP64:%.*]] = mul i64 [[TMP63]], 4 @@ -160,6 +160,8 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP83:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP84:%.*]] = mul i64 [[TMP83]], 16 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 16 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -262,8 +264,6 @@ define void @cond_memset(i32 %val, ptr noalias readonly %cond_ptr, ptr noalias % ; CHECK-NEXT: [[TMP81:%.*]] = mul i64 [[TMP80]], 12 ; CHECK-NEXT: [[TMP82:%.*]] = getelementptr i32, ptr [[TMP65]], i64 [[TMP81]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP82]], i32 4, [[TMP72]]) -; CHECK-NEXT: [[TMP83:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP84:%.*]] = mul i64 [[TMP83]], 16 ; CHECK-NEXT: [[INDEX_NEXT13]] = add i64 [[INDEX6]], [[TMP84]] ; CHECK-NEXT: [[TMP85:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP86:%.*]] = mul i64 [[TMP85]], 4 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll index 292bdd5c8be8..579e8d4ebb3c 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-tail-folding.ll @@ -18,6 +18,8 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -34,8 +36,6 @@ define void @simple_memset(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP10]] ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i32 0 ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP14]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP15:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -145,6 +145,8 @@ define void @simple_memcpy(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -162,8 +164,6 @@ define void @simple_memcpy(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[DST:%.*]], i64 [[TMP10]] ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP13]], i32 0 ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[WIDE_MASKED_LOAD]], ptr [[TMP14]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP17:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -222,6 +222,8 @@ define void @copy_stride4(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP4]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] ; CHECK-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 4 +; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 ; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; CHECK-NEXT: [[TMP10:%.*]] = sub i64 [[TMP2]], [[TMP9]] @@ -246,8 +248,6 @@ define void @copy_stride4(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[TMP19]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP20:%.*]] = getelementptr i32, ptr [[DST:%.*]], [[VEC_IND]] ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[WIDE_MASKED_GATHER]], [[TMP20]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP21:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP22:%.*]] = mul i64 [[TMP21]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP22]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP12]]) ; CHECK-NEXT: [[TMP23:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -303,6 +303,8 @@ define void @simple_gather_scatter(ptr noalias %dst, ptr noalias %src, ptr noali ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -321,8 +323,6 @@ define void @simple_gather_scatter(ptr noalias %dst, ptr noalias %src, ptr noali ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[TMP13]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[DST:%.*]], [[WIDE_MASKED_LOAD]] ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[WIDE_MASKED_GATHER]], [[TMP14]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP16]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP17:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -382,6 +382,8 @@ define void @uniform_load(ptr noalias %dst, ptr noalias readonly %src, i64 %n) # ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -399,8 +401,6 @@ define void @uniform_load(ptr noalias %dst, ptr noalias readonly %src, i64 %n) # ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[DST:%.*]], i64 [[TMP10]] ; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i32, ptr [[TMP12]], i32 0 ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP13]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP16:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -457,6 +457,8 @@ define void @cond_uniform_load(ptr noalias %dst, ptr noalias readonly %src, ptr ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -483,8 +485,6 @@ define void @cond_uniform_load(ptr noalias %dst, ptr noalias readonly %src, ptr ; CHECK-NEXT: [[TMP18:%.*]] = or [[TMP15]], [[TMP16]] ; CHECK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP17]], i32 0 ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[PREDPHI]], ptr [[TMP19]], i32 4, [[TMP18]]) -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP21]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP22:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -557,6 +557,8 @@ define void @uniform_store(ptr noalias %dst, ptr noalias readonly %src, i64 %n) ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[N:%.*]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[N]], [[TMP6]] @@ -574,8 +576,6 @@ define void @uniform_store(ptr noalias %dst, ptr noalias readonly %src, i64 %n) ; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i32, ptr [[TMP11]], i32 0 ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP14]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP15:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -629,6 +629,8 @@ define void @simple_fdiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -648,8 +650,6 @@ define void @simple_fdiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[WIDE_MASKED_LOAD2:%.*]] = call @llvm.masked.load.nxv4f32.p0(ptr [[TMP14]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP15:%.*]] = fdiv [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD2]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4f32.p0( [[TMP15]], ptr [[TMP14]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 4 ; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP17]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP18:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -709,6 +709,8 @@ define void @simple_idiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[TMP17]], 4 ; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -729,8 +731,6 @@ define void @simple_idiv(ptr noalias %dst, ptr noalias %src, i64 %n) #0 { ; CHECK-NEXT: [[TMP15:%.*]] = select [[ACTIVE_LANE_MASK]], [[WIDE_MASKED_LOAD2]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP16:%.*]] = udiv [[WIDE_MASKED_LOAD]], [[TMP15]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP16]], ptr [[TMP14]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP18:%.*]] = mul i64 [[TMP17]], 4 ; CHECK-NEXT: [[INDEX_NEXT3]] = add i64 [[INDEX1]], [[TMP18]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; CHECK-NEXT: [[TMP19:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -786,6 +786,8 @@ define void @simple_memset_trip1024(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -795,8 +797,6 @@ define void @simple_memset_trip1024(i32 %val, ptr %ptr, i64 %n) #0 { ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP4]] ; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i32 0 ; CHECK-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 4 -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 ; CHECK-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP8]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP22:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll index b36a10b46734..2cd52bfa87b9 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vector-reverse.ll @@ -23,6 +23,8 @@ define void @vector_reverse_f64(i64 %N, ptr noalias %a, ptr noalias %b) #0{ ; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP30:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP31:%.*]] = shl i64 [[TMP30]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -56,8 +58,6 @@ define void @vector_reverse_f64(i64 %N, ptr noalias %a, ptr noalias %b) #0{ ; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds double, ptr [[TMP19]], i64 [[TMP26]] ; CHECK-NEXT: [[TMP29:%.*]] = getelementptr inbounds double, ptr [[TMP28]], i64 [[TMP27]] ; CHECK-NEXT: store [[TMP18]], ptr [[TMP29]], align 8 -; CHECK-NEXT: [[TMP30:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP31:%.*]] = shl i64 [[TMP30]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP31]] ; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP32]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -125,6 +125,8 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 { ; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP6]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub nsw i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP34:%.*]] = shl i64 [[TMP33]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -158,8 +160,6 @@ define void @vector_reverse_i64(i64 %N, ptr %a, ptr %b) #0 { ; CHECK-NEXT: [[TMP31:%.*]] = getelementptr inbounds i64, ptr [[TMP22]], i64 [[TMP29]] ; CHECK-NEXT: [[TMP32:%.*]] = getelementptr inbounds i64, ptr [[TMP31]], i64 [[TMP30]] ; CHECK-NEXT: store [[TMP21]], ptr [[TMP32]], align 8 -; CHECK-NEXT: [[TMP33:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP34:%.*]] = shl i64 [[TMP33]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP34]] ; CHECK-NEXT: [[TMP35:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP35]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll index 31cf2c4e2db4..c3ace983fd91 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-vfabi.ll @@ -7,6 +7,8 @@ define void @test_big_little_params(ptr readonly %a, ptr readonly %b, ptr noalia ; CHECK-LABEL: define void @test_big_little_params ; CHECK-SAME: (ptr readonly [[A:%.*]], ptr readonly [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2 ; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 1025) ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: @@ -19,8 +21,6 @@ define void @test_big_little_params(ptr readonly %a, ptr readonly %b, ptr noalia ; CHECK-NEXT: [[TMP2:%.*]] = call @foo_vector( [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]], [[ACTIVE_LANE_MASK]]) ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[C]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP2]], ptr [[TMP3]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; CHECK-NEXT: [[TMP6:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 @@ -52,6 +52,8 @@ define void @test_little_big_params(ptr readonly %a, ptr readonly %b, ptr noalia ; CHECK-LABEL: define void @test_little_big_params ; CHECK-SAME: (ptr readonly [[A:%.*]], ptr readonly [[B:%.*]], ptr noalias [[C:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 1 ; CHECK-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 0, i64 1025) ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: @@ -64,8 +66,6 @@ define void @test_little_big_params(ptr readonly %a, ptr readonly %b, ptr noalia ; CHECK-NEXT: [[TMP2:%.*]] = call @bar_vector( [[WIDE_MASKED_LOAD]], [[WIDE_MASKED_LOAD1]], [[ACTIVE_LANE_MASK]]) ; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds double, ptr [[C]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP2]], ptr [[TMP3]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP5:%.*]] = shl nuw nsw i64 [[TMP4]], 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX_NEXT]], i64 1025) ; CHECK-NEXT: [[TMP6:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll index 62c778aca51d..140c469327bd 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-gep.ll @@ -9,6 +9,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NOT: LV: Found {{.*}} scalar instruction: %ptr.iv.2.next = getelementptr inbounds i8, ptr %ptr.iv.2, i64 1 ; ; CHECK: VPlan 'Initial VPlan for VF={vscale x 2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%N> = original trip-count ; CHECK-EMPTY: @@ -25,7 +26,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: WIDEN ir<%lv> = load ir<%ptr.iv.2> ; CHECK-NEXT: WIDEN ir<%add> = add ir<%lv>, ir<1> ; CHECK-NEXT: WIDEN store ir<%ptr.iv.2>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -49,6 +50,8 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias ; CHECK-NEXT: [[TMP4:%.*]] = mul i64 [[N_VEC]], 8 ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START_1:%.*]], i64 [[TMP4]] ; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[START_2:%.*]], i64 [[N_VEC]] +; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START_2]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] @@ -75,8 +78,6 @@ define void @pointer_induction_used_as_vector(ptr noalias %start.1, ptr noalias ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP18]], align 1 ; CHECK-NEXT: [[TMP19:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: store [[TMP19]], ptr [[TMP18]], align 1 -; CHECK-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP21]] ; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP10]] ; CHECK-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -142,6 +143,8 @@ define void @pointer_induction(ptr noalias %start, i64 %N) { ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP0]], [[TMP4]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP0]], [[N_MOD_VF]] ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[START:%.*]], i64 [[N_VEC]] +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[START]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] @@ -162,8 +165,6 @@ define void @pointer_induction(ptr noalias %start, i64 %N) { ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP14]], align 1 ; CHECK-NEXT: [[TMP15:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: store [[TMP15]], ptr [[TMP14]], align 1 -; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX2]], [[TMP17]] ; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP8]] ; CHECK-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll index 9cd2781ae235..c4386c3ce5d1 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/sve-widen-phi.ll @@ -27,6 +27,8 @@ define void @widen_ptr_phi_unrolled(ptr noalias nocapture %a, ptr noalias nocapt ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[DOTNEG]], [[N]] ; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[N_VEC]], 3 ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[C:%.*]], i64 [[TMP3]] +; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP26:%.*]] = shl nuw nsw i64 [[TMP25]], 3 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -61,8 +63,6 @@ define void @widen_ptr_phi_unrolled(ptr noalias nocapture %a, ptr noalias nocapt ; CHECK-NEXT: [[TMP23:%.*]] = shl nuw nsw i64 [[TMP22]], 2 ; CHECK-NEXT: [[TMP24:%.*]] = getelementptr inbounds i32, ptr [[TMP21]], i64 [[TMP23]] ; CHECK-NEXT: store [[TMP20]], ptr [[TMP24]], align 4 -; CHECK-NEXT: [[TMP25:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP26:%.*]] = shl nuw nsw i64 [[TMP25]], 3 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP26]] ; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -143,6 +143,8 @@ define void @widen_2ptrs_phi_unrolled(ptr noalias nocapture %dst, ptr noalias no ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[SRC:%.*]], i64 [[TMP3]] ; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 2 ; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[DST:%.*]], i64 [[TMP4]] +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 3 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -162,8 +164,6 @@ define void @widen_2ptrs_phi_unrolled(ptr noalias nocapture %dst, ptr noalias no ; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP12]], 2 ; CHECK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[NEXT_GEP5]], i64 [[TMP13]] ; CHECK-NEXT: store [[TMP11]], ptr [[TMP14]], align 4 -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = shl nuw nsw i64 [[TMP15]], 3 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] @@ -234,6 +234,8 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 { ; CHECK-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[A:%.*]], i64 [[TMP3]] ; CHECK-NEXT: [[TMP4:%.*]] = shl i64 [[N_VEC]], 3 ; CHECK-NEXT: [[IND_END2:%.*]] = getelementptr i8, ptr [[B:%.*]], i64 [[TMP4]] +; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP12]], 1 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[A]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] @@ -250,8 +252,6 @@ define i32 @pointer_iv_mixed(ptr noalias %a, ptr noalias %b, i64 %n) #0 { ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP10]], align 8 ; CHECK-NEXT: [[TMP11]] = add [[WIDE_LOAD]], [[VEC_PHI]] ; CHECK-NEXT: store [[TMP8]], ptr [[NEXT_GEP]], align 8 -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = shl nuw nsw i64 [[TMP12]], 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] ; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP6]] ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -310,6 +310,8 @@ define void @phi_used_in_vector_compare_and_scalar_indvar_update_and_store(ptr % ; CHECK-NEXT: entry: ; CHECK-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 1 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[PTR:%.*]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] @@ -322,8 +324,6 @@ define void @phi_used_in_vector_compare_and_scalar_indvar_update_and_store(ptr % ; CHECK-NEXT: [[TMP4:%.*]] = icmp ne [[TMP3]], zeroinitializer ; CHECK-NEXT: [[TMP5:%.*]] = extractelement [[TMP3]], i64 0 ; CHECK-NEXT: call void @llvm.masked.store.nxv2i16.p0( zeroinitializer, ptr [[TMP5]], i32 2, [[TMP4]]) -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl nuw nsw i64 [[TMP6]], 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP1]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll index eaabc263913a..a35c0a4595d9 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/synthesize-mask-for-call.ll @@ -10,6 +10,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-LABEL: LV: Checking a loop in 'test_v4_v4m' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -25,7 +26,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: REPLICATE ir<%call> = call @foo(ir<%load>) ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -36,6 +37,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -51,7 +53,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask) ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -67,6 +69,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-LABEL: LV: Checking a loop in 'test_v2_v4m' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -82,7 +85,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask) ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXST:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXST:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -93,6 +96,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -108,7 +112,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>, ir) (using library function: foo_vector_fixed4_mask) ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -123,6 +127,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-LABEL: LV: Checking a loop in 'test_v2_v4' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -138,7 +143,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed2_nomask) ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -149,6 +154,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -164,7 +170,7 @@ target triple = "aarch64-unknown-linux-gnu" ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @foo(ir<%load>) (using library function: foo_vector_fixed4_nomask) ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%a>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll index 814d3ba53050..94f24fea3609 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/tail-folding-styles.ll @@ -22,6 +22,8 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; NONE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; NONE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[UMAX]], [[TMP3]] ; NONE-NEXT: [[N_VEC:%.*]] = sub i64 [[UMAX]], [[N_MOD_VF]] +; NONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; NONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 ; NONE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; NONE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; NONE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -31,8 +33,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; NONE-NEXT: [[TMP5:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP4]] ; NONE-NEXT: [[TMP6:%.*]] = getelementptr i32, ptr [[TMP5]], i32 0 ; NONE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 4 -; NONE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; NONE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 ; NONE-NEXT: [[INDEX_NEXT2]] = add nuw i64 [[INDEX1]], [[TMP8]] ; NONE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] ; NONE-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -69,6 +69,8 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] ; DATA-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] ; DATA-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; DATA-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; DATA-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; DATA-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; DATA-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; DATA-NEXT: br label [[VECTOR_BODY:%.*]] @@ -79,8 +81,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP9]] ; DATA-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[TMP10]], i32 0 ; DATA-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP11]], i32 4, [[ACTIVE_LANE_MASK]]) -; DATA-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; DATA-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; DATA-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP13]] ; DATA-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT2]], [[N_VEC]] ; DATA-NEXT: br i1 [[TMP14]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -119,6 +119,8 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA_NO_LANEMASK-NEXT: [[TRIP_COUNT_MINUS_1:%.*]] = sub i64 [[UMAX]], 1 ; DATA_NO_LANEMASK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[TRIP_COUNT_MINUS_1]], i64 0 ; DATA_NO_LANEMASK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer +; DATA_NO_LANEMASK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; DATA_NO_LANEMASK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; DATA_NO_LANEMASK-NEXT: [[BROADCAST_SPLATINSERT4:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; DATA_NO_LANEMASK-NEXT: [[BROADCAST_SPLAT5:%.*]] = shufflevector [[BROADCAST_SPLATINSERT4]], poison, zeroinitializer ; DATA_NO_LANEMASK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -134,8 +136,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA_NO_LANEMASK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP9]] ; DATA_NO_LANEMASK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP13]], i32 0 ; DATA_NO_LANEMASK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT5]], ptr [[TMP14]], i32 4, [[TMP12]]) -; DATA_NO_LANEMASK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; DATA_NO_LANEMASK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; DATA_NO_LANEMASK-NEXT: [[INDEX_NEXT6]] = add i64 [[INDEX1]], [[TMP16]] ; DATA_NO_LANEMASK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT6]], [[N_VEC]] ; DATA_NO_LANEMASK-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -171,6 +171,8 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA_AND_CONTROL-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP8]] ; DATA_AND_CONTROL-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP5]] ; DATA_AND_CONTROL-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; DATA_AND_CONTROL-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; DATA_AND_CONTROL-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; DATA_AND_CONTROL-NEXT: [[ACTIVE_LANE_MASK_ENTRY:%.*]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 0, i64 [[UMAX]]) ; DATA_AND_CONTROL-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; DATA_AND_CONTROL-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer @@ -182,8 +184,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA_AND_CONTROL-NEXT: [[TMP10:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP9]] ; DATA_AND_CONTROL-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[TMP10]], i32 0 ; DATA_AND_CONTROL-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP11]], i32 4, [[ACTIVE_LANE_MASK]]) -; DATA_AND_CONTROL-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; DATA_AND_CONTROL-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; DATA_AND_CONTROL-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP13]] ; DATA_AND_CONTROL-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX_NEXT2]], i64 [[UMAX]]) ; DATA_AND_CONTROL-NEXT: [[TMP14:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) @@ -217,6 +217,8 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 [[UMAX]], [[TMP4]] ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() +; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 4 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP7:%.*]] = sub i64 [[UMAX]], [[TMP6]] @@ -233,8 +235,6 @@ define void @simple_memset_tailfold(i32 %val, ptr %ptr, i64 %n) "target-features ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP11:%.*]] = getelementptr i32, ptr [[PTR:%.*]], i64 [[TMP10]] ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[TMP11]], i32 0 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[BROADCAST_SPLAT]], ptr [[TMP12]], i32 4, [[ACTIVE_LANE_MASK]]) -; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP14:%.*]] = mul i64 [[TMP13]], 4 ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[INDEX_NEXT2]] = add i64 [[INDEX1]], [[TMP14]] ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv4i1.i64(i64 [[INDEX1]], i64 [[TMP9]]) ; DATA_AND_CONTROL_NO_RT_CHECK-NEXT: [[TMP15:%.*]] = xor [[ACTIVE_LANE_MASK_NEXT]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll index f49b3714062d..dec3c286345a 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/type-shrinkage-zext-costs.ll @@ -32,18 +32,18 @@ define void @zext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 8 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 8 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 1 -; CHECK-NEXT: [[TMP8:%.*]] = zext [[WIDE_LOAD]] to -; CHECK-NEXT: [[TMP9:%.*]] = add [[TMP8]], trunc ( shufflevector ( insertelement ( poison, i32 2, i64 0), poison, zeroinitializer) to ) -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]] -; CHECK-NEXT: store [[TMP9]], ptr [[TMP10]], align 2 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP9]], align 1 +; CHECK-NEXT: [[TMP10:%.*]] = zext [[WIDE_LOAD]] to +; CHECK-NEXT: [[TMP11:%.*]] = add [[TMP10]], trunc ( shufflevector ( insertelement ( poison, i32 2, i64 0), poison, zeroinitializer) to ) +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]] +; CHECK-NEXT: store [[TMP11]], ptr [[TMP12]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -115,18 +115,18 @@ define void @sext_i8_i16(ptr noalias nocapture readonly %p, ptr noalias nocaptur ; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 8 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[TMP2]], [[TMP6]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[TMP2]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 8 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 1 -; CHECK-NEXT: [[TMP8:%.*]] = sext [[WIDE_LOAD]] to -; CHECK-NEXT: [[TMP9:%.*]] = add [[TMP8]], trunc ( shufflevector ( insertelement ( poison, i32 2, i64 0), poison, zeroinitializer) to ) -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]] -; CHECK-NEXT: store [[TMP9]], ptr [[TMP10]], align 2 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 8 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[P]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP9]], align 1 +; CHECK-NEXT: [[TMP10:%.*]] = sext [[WIDE_LOAD]] to +; CHECK-NEXT: [[TMP11:%.*]] = add [[TMP10]], trunc ( shufflevector ( insertelement ( poison, i32 2, i64 0), poison, zeroinitializer) to ) +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i16, ptr [[Q]], i64 [[INDEX]] +; CHECK-NEXT: store [[TMP11]], ptr [[TMP12]], align 2 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll index 51465dbf13e2..bda20ae18bc9 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/uniform-args-call-variants.ll @@ -9,6 +9,8 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; CHECK-LABEL: define void @test_uniform ; CHECK-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i64 [[UNIFORM:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 1 ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 1 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP1]]) @@ -22,8 +24,6 @@ define void @test_uniform(ptr noalias %dst, ptr readonly %src, i64 %uniform , i6 ; CHECK-NEXT: [[TMP4:%.*]] = call @foo_uniform( [[WIDE_MASKED_LOAD]], i64 [[UNIFORM]], [[ACTIVE_LANE_MASK]]) ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP4]], ptr [[TMP5]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP2]]) ; CHECK-NEXT: [[TMP8:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 @@ -53,6 +53,8 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; CHECK-LABEL: define void @test_uniform_smaller_scalar ; CHECK-SAME: (ptr noalias [[DST:%.*]], ptr readonly [[SRC:%.*]], i32 [[UNIFORM:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { ; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 1 ; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.vscale.i64() ; CHECK-NEXT: [[TMP1:%.*]] = shl i64 [[TMP0]], 1 ; CHECK-NEXT: [[TMP2:%.*]] = call i64 @llvm.usub.sat.i64(i64 [[N]], i64 [[TMP1]]) @@ -66,8 +68,6 @@ define void @test_uniform_smaller_scalar(ptr noalias %dst, ptr readonly %src, i3 ; CHECK-NEXT: [[TMP4:%.*]] = call @bar_uniform( [[WIDE_MASKED_LOAD]], i32 [[UNIFORM]], [[ACTIVE_LANE_MASK]]) ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds double, ptr [[DST]], i64 [[INDEX]] ; CHECK-NEXT: call void @llvm.masked.store.nxv2f64.p0( [[TMP4]], ptr [[TMP5]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = shl i64 [[TMP6]], 1 ; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[ACTIVE_LANE_MASK_NEXT]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[INDEX]], i64 [[TMP2]]) ; CHECK-NEXT: [[TMP8:%.*]] = extractelement [[ACTIVE_LANE_MASK_NEXT]], i64 0 diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/vector-call-linear-args.ll b/llvm/test/Transforms/LoopVectorize/AArch64/vector-call-linear-args.ll index cd133371f66c..876d58131bd7 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/vector-call-linear-args.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/vector-call-linear-args.ll @@ -1,4 +1,4 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call" --version 2 +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "call.*(foo|bar|baz|quux)" --version 2 ; RUN: opt < %s -passes=loop-vectorize -force-vector-interleave=1 -S | FileCheck %s --check-prefixes=NEON ; RUN: opt < %s -mattr=+sve -passes=loop-vectorize -force-vector-interleave=1 -S | FileCheck %s --check-prefixes=SVE_OR_NEON ; RUN: opt < %s -mattr=+sve -passes=loop-vectorize -force-vector-interleave=1 -S -prefer-predicate-over-epilogue=predicate-dont-vectorize | FileCheck %s --check-prefixes=SVE_TF @@ -10,15 +10,18 @@ target triple = "aarch64-unknown-linux-gnu" define void @test_linear8(ptr noalias %a, ptr readnone %b, i64 %n) { ; NEON-LABEL: define void @test_linear8 ; NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) { +; NEON: [[TMP3:%.*]] = call <2 x i64> @vec_foo_linear8_nomask_neon(ptr [[TMP2:%.*]]) ; NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR0:[0-9]+]] ; ; SVE_OR_NEON-LABEL: define void @test_linear8 ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR1:[0-9]+]] +; SVE_OR_NEON: [[TMP13:%.*]] = call @vec_foo_linear8_nomask_sve(ptr [[TMP12:%.*]]) +; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR2:[0-9]+]] ; ; SVE_TF-LABEL: define void @test_linear8 ; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] { -; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR1:[0-9]+]] +; SVE_TF: [[TMP19:%.*]] = call @vec_foo_linear8_mask_sve(ptr [[TMP18:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3:[0-9]+]] ; entry: br label %for.body @@ -40,15 +43,17 @@ for.cond.cleanup: define void @test_vector_linear4(ptr noalias %a, ptr readnone %b, ptr readonly %c, i64 %n) { ; NEON-LABEL: define void @test_vector_linear4 ; NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly [[C:%.*]], i64 [[N:%.*]]) { +; NEON: [[TMP5:%.*]] = call <4 x i32> @vec_baz_vector_linear4_nomask_neon(<4 x i32> [[WIDE_LOAD:%.*]], ptr [[TMP4:%.*]]) ; NEON: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]]) #[[ATTR1:[0-9]+]] ; ; SVE_OR_NEON-LABEL: define void @test_vector_linear4 ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]]) #[[ATTR2:[0-9]+]] +; SVE_OR_NEON: [[TMP15:%.*]] = call @vec_baz_vector_linear4_nomask_sve( [[WIDE_LOAD:%.*]], ptr [[TMP14:%.*]]) +; SVE_OR_NEON: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]]) #[[ATTR3:[0-9]+]] ; ; SVE_TF-LABEL: define void @test_vector_linear4 ; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_TF: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]]) #[[ATTR2:[0-9]+]] +; SVE_TF: [[DATA:%.*]] = call i32 @baz(i32 [[INPUT:%.*]], ptr [[GEPB:%.*]]) #[[ATTR4:[0-9]+]] ; entry: br label %for.body @@ -76,11 +81,11 @@ define void @test_linear8_bad_stride(ptr noalias %a, ptr readnone %b, i64 %n) { ; ; SVE_OR_NEON-LABEL: define void @test_linear8_bad_stride ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3:[0-9]+]] +; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR4:[0-9]+]] ; ; SVE_TF-LABEL: define void @test_linear8_bad_stride ; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3:[0-9]+]] +; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR5:[0-9]+]] ; entry: br label %for.body @@ -102,15 +107,17 @@ for.cond.cleanup: define void @test_linear16_wide_stride(ptr noalias %a, ptr readnone %b, i64 %n) { ; NEON-LABEL: define void @test_linear16_wide_stride ; NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) { +; NEON: [[TMP4:%.*]] = call <2 x i64> @vec_foo_linear16_nomask_neon(ptr [[TMP3:%.*]]) ; NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR2]] ; ; SVE_OR_NEON-LABEL: define void @test_linear16_wide_stride ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3]] +; SVE_OR_NEON: [[TMP14:%.*]] = call @vec_foo_linear16_nomask_sve(ptr [[TMP13:%.*]]) +; SVE_OR_NEON: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR4]] ; ; SVE_TF-LABEL: define void @test_linear16_wide_stride ; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR3]] +; SVE_TF: [[DATA:%.*]] = call i64 @foo(ptr [[GEPB:%.*]]) #[[ATTR5]] ; entry: br label %for.body @@ -133,15 +140,18 @@ for.cond.cleanup: define void @test_linear4_linear8(ptr noalias %a, ptr readnone %b, ptr readonly %c, i64 %n) { ; NEON-LABEL: define void @test_linear4_linear8 ; NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly [[C:%.*]], i64 [[N:%.*]]) { +; NEON: [[TMP5:%.*]] = call <4 x i32> @vec_quux_linear4_linear8_nomask_neon(ptr [[TMP3:%.*]], ptr [[TMP4:%.*]]) ; NEON: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]]) #[[ATTR3:[0-9]+]] ; ; SVE_OR_NEON-LABEL: define void @test_linear4_linear8 ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]]) #[[ATTR4:[0-9]+]] +; SVE_OR_NEON: [[TMP15:%.*]] = call @vec_quux_linear4_linear8_mask_sve(ptr [[TMP13:%.*]], ptr [[TMP14:%.*]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) +; SVE_OR_NEON: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]]) #[[ATTR5:[0-9]+]] ; ; SVE_TF-LABEL: define void @test_linear4_linear8 ; SVE_TF-SAME: (ptr noalias [[A:%.*]], ptr readnone [[B:%.*]], ptr readonly [[C:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_TF: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]]) #[[ATTR4:[0-9]+]] +; SVE_TF: [[TMP21:%.*]] = call @vec_quux_linear4_linear8_mask_sve(ptr [[TMP19:%.*]], ptr [[TMP20:%.*]], [[ACTIVE_LANE_MASK:%.*]]) +; SVE_TF: [[DATA:%.*]] = call i32 @quux(ptr [[GEPC:%.*]], ptr [[GEPB:%.*]]) #[[ATTR6:[0-9]+]] ; entry: br label %for.body @@ -164,15 +174,17 @@ for.cond.cleanup: define void @test_linear3_non_ptr(ptr noalias %a, i64 %n) { ; NEON-LABEL: define void @test_linear3_non_ptr ; NEON-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) { +; NEON: [[TMP3:%.*]] = call <4 x i32> @vec_bar_linear3_nomask_neon(i32 [[TMP2:%.*]]) ; NEON: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]]) #[[ATTR4:[0-9]+]] ; ; SVE_OR_NEON-LABEL: define void @test_linear3_non_ptr ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]]) #[[ATTR5:[0-9]+]] +; SVE_OR_NEON: [[TMP13:%.*]] = call @vec_bar_linear3_nomask_sve(i32 [[TMP12:%.*]]) +; SVE_OR_NEON: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]]) #[[ATTR6:[0-9]+]] ; ; SVE_TF-LABEL: define void @test_linear3_non_ptr ; SVE_TF-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_TF: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]]) #[[ATTR5:[0-9]+]] +; SVE_TF: [[DATA:%.*]] = call i32 @bar(i32 [[TREBLED:%.*]]) #[[ATTR7:[0-9]+]] ; entry: br label %for.body @@ -195,15 +207,17 @@ for.cond.cleanup: define void @test_linearn5_non_ptr_neg_stride(ptr noalias %a, i64 %n) { ; NEON-LABEL: define void @test_linearn5_non_ptr_neg_stride ; NEON-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) { +; NEON: [[TMP3:%.*]] = call <4 x i32> @vec_bar_linearn5_nomask_neon(i32 [[TMP2:%.*]]) ; NEON: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]]) #[[ATTR5:[0-9]+]] ; ; SVE_OR_NEON-LABEL: define void @test_linearn5_non_ptr_neg_stride ; SVE_OR_NEON-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_OR_NEON: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]]) #[[ATTR6:[0-9]+]] +; SVE_OR_NEON: [[TMP13:%.*]] = call @vec_bar_linearn5_nomask_sve(i32 [[TMP12:%.*]]) +; SVE_OR_NEON: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]]) #[[ATTR7:[0-9]+]] ; ; SVE_TF-LABEL: define void @test_linearn5_non_ptr_neg_stride ; SVE_TF-SAME: (ptr noalias [[A:%.*]], i64 [[N:%.*]]) #[[ATTR0]] { -; SVE_TF: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]]) #[[ATTR6:[0-9]+]] +; SVE_TF: [[DATA:%.*]] = call i32 @bar(i32 [[NEGSTRIDE:%.*]]) #[[ATTR8:[0-9]+]] ; entry: br label %for.body diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll index f82eb9c7822f..a3a2223a2528 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/widen-call-with-intrinsic-or-libfunc.ll @@ -7,7 +7,9 @@ target triple = "arm64-apple-ios" ; CHECK-LABEL: LV: Checking a loop in 'test' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count + ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: @@ -23,7 +25,7 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using library function: __simd_sin_v2f64) ; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> ; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -34,6 +36,7 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: } ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1024> = original trip-count ; CHECK-EMPTY: @@ -50,7 +53,7 @@ target triple = "arm64-apple-ios" ; CHECK-NEXT: WIDEN-CALL ir<%s> = call @llvm.sin.f64(ir<%conv>) (using vector intrinsic) ; CHECK-NEXT: REPLICATE ir<%gep.dst> = getelementptr inbounds ir<%dst>, vp<[[STEPS]]> ; CHECK-NEXT: REPLICATE store ir<%s>, ir<%gep.dst> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll b/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll index 7d095206c606..4a2f9d07ed91 100644 --- a/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll +++ b/llvm/test/Transforms/LoopVectorize/AArch64/wider-VF-for-callinst.ll @@ -16,6 +16,8 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #1 { ; WIDE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; WIDE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; WIDE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; WIDE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; WIDE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; WIDE-NEXT: br label [[VECTOR_BODY:%.*]] ; WIDE: vector.body: ; WIDE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -25,8 +27,6 @@ define void @test_widen(ptr noalias %a, ptr readnone %b) #1 { ; WIDE-NEXT: [[TMP6:%.*]] = call @foo_vector( [[TMP5]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; WIDE-NEXT: [[TMP7:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] ; WIDE-NEXT: store [[TMP6]], ptr [[TMP7]], align 4 -; WIDE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; WIDE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; WIDE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; WIDE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; WIDE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/Hexagon/maximum-vf-crash.ll b/llvm/test/Transforms/LoopVectorize/Hexagon/maximum-vf-crash.ll index 4c7a7d31c5e7..4966ddd29949 100644 --- a/llvm/test/Transforms/LoopVectorize/Hexagon/maximum-vf-crash.ll +++ b/llvm/test/Transforms/LoopVectorize/Hexagon/maximum-vf-crash.ll @@ -1,4 +1,4 @@ -; RUN: opt -march=hexagon -hexagon-autohvx -passes=loop-vectorize -S < %s 2>&1 | FileCheck %s +; RUN: opt -mtriple=hexagon -hexagon-autohvx -passes=loop-vectorize -S < %s 2>&1 | FileCheck %s ; Check that we don't crash. @@ -6,7 +6,6 @@ ; CHECK: vector.body target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048" -target triple = "hexagon" ; Function Attrs: optsize define i32 @f() #0 { diff --git a/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll b/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll index 4ff7ad9ab439..1ac556a2fda3 100644 --- a/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll +++ b/llvm/test/Transforms/LoopVectorize/Hexagon/minimum-vf.ll @@ -1,4 +1,4 @@ -; RUN: opt -march=hexagon -passes=loop-vectorize -hexagon-autohvx -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s +; RUN: opt -mtriple=hexagon -passes=loop-vectorize -hexagon-autohvx -debug-only=loop-vectorize -disable-output < %s 2>&1 | FileCheck %s ; REQUIRES: asserts ; Check that TTI::getMinimumVF works. The calculated MaxVF was based on the @@ -6,7 +6,6 @@ ; CHECK: LV: Overriding calculated MaxVF({{[0-9]+}}) with target's minimum: 64 target datalayout = "e-m:e-p:32:32:32-a:0-n16:32-i64:64:64-i32:32:32-i16:16:16-i1:8:8-f32:32:32-f64:64:64-v32:32:32-v64:64:64-v512:512:512-v1024:1024:1024-v2048:2048:2048" -target triple = "hexagon" %s.0 = type { ptr, i32, i32, i32, i32 } diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll b/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll index 4b93ea30cf25..db0be4c62815 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/defaults.ll @@ -22,20 +22,20 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = add [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: store [[TMP9]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -85,22 +85,22 @@ define i64 @vector_add_reduce(ptr noalias nocapture %a) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP7:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7]] = add [[VEC_PHI]], [[WIDE_LOAD]] -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9]] = add [[VEC_PHI]], [[WIDE_LOAD]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64( [[TMP7]]) +; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64( [[TMP9]]) ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 1024, [[N_VEC]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll index a6ab8d1a1f35..1f1891ee4548 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/divrem.ll @@ -19,20 +19,20 @@ define void @vector_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = udiv [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = udiv [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: store [[TMP9]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -123,20 +123,20 @@ define void @vector_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = sdiv [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = sdiv [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: store [[TMP9]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: @@ -227,20 +227,20 @@ define void @vector_urem(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = urem [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = urem [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: store [[TMP9]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: @@ -331,20 +331,20 @@ define void @vector_srem(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = srem [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = srem [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: store [[TMP9]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: middle.block: @@ -435,24 +435,24 @@ define void @predicated_udiv(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[BROADCAST_SPLAT]], zeroinitializer -; CHECK-NEXT: [[TMP8:%.*]] = select [[TMP7]], [[BROADCAST_SPLAT]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = udiv [[WIDE_LOAD]], [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP7]], [[TMP9]], [[WIDE_LOAD]] -; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne [[BROADCAST_SPLAT]], zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = select [[TMP9]], [[BROADCAST_SPLAT]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = udiv [[WIDE_LOAD]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = xor [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP9]], [[TMP11]], [[WIDE_LOAD]] +; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: middle.block: @@ -569,24 +569,24 @@ define void @predicated_sdiv(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[BROADCAST_SPLAT]], zeroinitializer -; CHECK-NEXT: [[TMP8:%.*]] = select [[TMP7]], [[BROADCAST_SPLAT]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = sdiv [[WIDE_LOAD]], [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP7]], [[TMP9]], [[WIDE_LOAD]] -; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne [[BROADCAST_SPLAT]], zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = select [[TMP9]], [[BROADCAST_SPLAT]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = sdiv [[WIDE_LOAD]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = xor [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP9]], [[TMP11]], [[WIDE_LOAD]] +; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] ; CHECK: middle.block: @@ -703,21 +703,21 @@ define void @predicated_udiv_by_constant(ptr noalias nocapture %a, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 42, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP8:%.*]] = udiv [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 27, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP7]], [[TMP8]], [[WIDE_LOAD]] -; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 42, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP10:%.*]] = udiv [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 27, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = xor [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP9]], [[TMP10]], [[WIDE_LOAD]] +; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] ; CHECK: middle.block: @@ -830,21 +830,21 @@ define void @predicated_sdiv_by_constant(ptr noalias nocapture %a, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 42, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP8:%.*]] = sdiv [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 27, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP7]], [[TMP8]], [[WIDE_LOAD]] -; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 42, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP10:%.*]] = sdiv [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 27, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = xor [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP9]], [[TMP10]], [[WIDE_LOAD]] +; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP8]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] ; CHECK: middle.block: @@ -957,22 +957,22 @@ define void @predicated_sdiv_by_minus_one(ptr noalias nocapture %a, i64 %n) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 16 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 16 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 1 -; CHECK-NEXT: [[TMP7:%.*]] = icmp ne [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i8 -128, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP8:%.*]] = select [[TMP7]], shufflevector ( insertelement ( poison, i8 -1, i64 0), poison, zeroinitializer), shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = sdiv [[WIDE_LOAD]], [[TMP8]] -; CHECK-NEXT: [[TMP10:%.*]] = xor [[TMP7]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP7]], [[TMP9]], [[WIDE_LOAD]] -; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP6]], align 1 -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 16 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[A:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 1 +; CHECK-NEXT: [[TMP9:%.*]] = icmp ne [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i8 -128, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP10:%.*]] = select [[TMP9]], shufflevector ( insertelement ( poison, i8 -1, i64 0), poison, zeroinitializer), shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = sdiv [[WIDE_LOAD]], [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = xor [[TMP9]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[PREDPHI:%.*]] = select [[TMP9]], [[TMP11]], [[WIDE_LOAD]] +; CHECK-NEXT: store [[PREDPHI]], ptr [[TMP8]], align 1 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll index 1310ed3618b2..57e1dc9051f4 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/inloop-reduction.ll @@ -21,23 +21,23 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; OUTLOOP-NEXT: [[TMP3:%.*]] = mul i32 [[TMP2]], 4 ; OUTLOOP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP3]] ; OUTLOOP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]] +; OUTLOOP-NEXT: [[TMP4:%.*]] = call i32 @llvm.vscale.i32() +; OUTLOOP-NEXT: [[TMP5:%.*]] = mul i32 [[TMP4]], 4 ; OUTLOOP-NEXT: br label [[VECTOR_BODY:%.*]] ; OUTLOOP: vector.body: ; OUTLOOP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; OUTLOOP-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP8:%.*]], [[VECTOR_BODY]] ] -; OUTLOOP-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 0 -; OUTLOOP-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP4]] -; OUTLOOP-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[TMP5]], i32 0 -; OUTLOOP-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 2 -; OUTLOOP-NEXT: [[TMP7:%.*]] = sext [[WIDE_LOAD]] to -; OUTLOOP-NEXT: [[TMP8]] = add [[VEC_PHI]], [[TMP7]] -; OUTLOOP-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() -; OUTLOOP-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 4 -; OUTLOOP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP10]] +; OUTLOOP-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP10:%.*]], [[VECTOR_BODY]] ] +; OUTLOOP-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 0 +; OUTLOOP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP6]] +; OUTLOOP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[TMP7]], i32 0 +; OUTLOOP-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 2 +; OUTLOOP-NEXT: [[TMP9:%.*]] = sext [[WIDE_LOAD]] to +; OUTLOOP-NEXT: [[TMP10]] = add [[VEC_PHI]], [[TMP9]] +; OUTLOOP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]] ; OUTLOOP-NEXT: [[TMP11:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; OUTLOOP-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; OUTLOOP: middle.block: -; OUTLOOP-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP8]]) +; OUTLOOP-NEXT: [[TMP12:%.*]] = call i32 @llvm.vector.reduce.add.nxv4i32( [[TMP10]]) ; OUTLOOP-NEXT: [[CMP_N:%.*]] = icmp eq i32 [[N]], [[N_VEC]] ; OUTLOOP-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; OUTLOOP: scalar.ph: @@ -75,20 +75,20 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; INLOOP-NEXT: [[TMP3:%.*]] = mul i32 [[TMP2]], 8 ; INLOOP-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], [[TMP3]] ; INLOOP-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]] +; INLOOP-NEXT: [[TMP4:%.*]] = call i32 @llvm.vscale.i32() +; INLOOP-NEXT: [[TMP5:%.*]] = mul i32 [[TMP4]], 8 ; INLOOP-NEXT: br label [[VECTOR_BODY:%.*]] ; INLOOP: vector.body: ; INLOOP-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; INLOOP-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ] -; INLOOP-NEXT: [[TMP4:%.*]] = add i32 [[INDEX]], 0 -; INLOOP-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP4]] -; INLOOP-NEXT: [[TMP6:%.*]] = getelementptr inbounds i16, ptr [[TMP5]], i32 0 -; INLOOP-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 2 -; INLOOP-NEXT: [[TMP7:%.*]] = sext [[WIDE_LOAD]] to -; INLOOP-NEXT: [[TMP8:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32( [[TMP7]]) -; INLOOP-NEXT: [[TMP9]] = add i32 [[TMP8]], [[VEC_PHI]] -; INLOOP-NEXT: [[TMP10:%.*]] = call i32 @llvm.vscale.i32() -; INLOOP-NEXT: [[TMP11:%.*]] = mul i32 [[TMP10]], 8 -; INLOOP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP11]] +; INLOOP-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ] +; INLOOP-NEXT: [[TMP6:%.*]] = add i32 [[INDEX]], 0 +; INLOOP-NEXT: [[TMP7:%.*]] = getelementptr inbounds i16, ptr [[X:%.*]], i32 [[TMP6]] +; INLOOP-NEXT: [[TMP8:%.*]] = getelementptr inbounds i16, ptr [[TMP7]], i32 0 +; INLOOP-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 2 +; INLOOP-NEXT: [[TMP9:%.*]] = sext [[WIDE_LOAD]] to +; INLOOP-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.nxv8i32( [[TMP9]]) +; INLOOP-NEXT: [[TMP11]] = add i32 [[TMP10]], [[VEC_PHI]] +; INLOOP-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP5]] ; INLOOP-NEXT: [[TMP12:%.*]] = icmp eq i32 [[INDEX_NEXT]], [[N_VEC]] ; INLOOP-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; INLOOP: middle.block: @@ -96,7 +96,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; INLOOP-NEXT: br i1 [[CMP_N]], label [[FOR_COND_CLEANUP_LOOPEXIT:%.*]], label [[SCALAR_PH]] ; INLOOP: scalar.ph: ; INLOOP-NEXT: [[BC_RESUME_VAL:%.*]] = phi i32 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[FOR_BODY_PREHEADER]] ] -; INLOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ] +; INLOOP-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ] ; INLOOP-NEXT: br label [[FOR_BODY:%.*]] ; INLOOP: for.body: ; INLOOP-NEXT: [[I_08:%.*]] = phi i32 [ [[INC:%.*]], [[FOR_BODY]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] @@ -109,7 +109,7 @@ define i32 @add_i16_i32(ptr nocapture readonly %x, i32 %n) { ; INLOOP-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INC]], [[N]] ; INLOOP-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; INLOOP: for.cond.cleanup.loopexit: -; INLOOP-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP9]], [[MIDDLE_BLOCK]] ] +; INLOOP-NEXT: [[ADD_LCSSA:%.*]] = phi i32 [ [[ADD]], [[FOR_BODY]] ], [ [[TMP11]], [[MIDDLE_BLOCK]] ] ; INLOOP-NEXT: br label [[FOR_COND_CLEANUP]] ; INLOOP: for.cond.cleanup: ; INLOOP-NEXT: [[R_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD_LCSSA]], [[FOR_COND_CLEANUP_LOOPEXIT]] ] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll index 29ef7364b821..a9cd91b5a8cb 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/interleaved-accesses.ll @@ -248,6 +248,8 @@ define void @load_store_factor3_i64(ptr %p) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 ; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; CHECK-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -276,8 +278,6 @@ define void @load_store_factor3_i64(ptr %p) { ; CHECK-NEXT: [[WIDE_MASKED_GATHER2:%.*]] = call @llvm.masked.gather.nxv2i64.nxv2p0( [[TMP17]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; CHECK-NEXT: [[TMP18:%.*]] = add [[WIDE_MASKED_GATHER2]], shufflevector ( insertelement ( poison, i64 3, i64 0), poison, zeroinitializer) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0( [[TMP18]], [[TMP17]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP20]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -353,6 +353,8 @@ define void @load_store_factor8(ptr %p) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP34:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP34]], 2 ; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; CHECK-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -406,8 +408,6 @@ define void @load_store_factor8(ptr %p) { ; CHECK-NEXT: [[WIDE_MASKED_GATHER7:%.*]] = call @llvm.masked.gather.nxv2i64.nxv2p0( [[TMP32]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; CHECK-NEXT: [[TMP33:%.*]] = add [[WIDE_MASKED_GATHER7]], shufflevector ( insertelement ( poison, i64 8, i64 0), poison, zeroinitializer) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0( [[TMP33]], [[TMP32]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP34:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP35:%.*]] = mul i64 [[TMP34]], 2 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP35]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP36:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll b/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll index 072cef66c00f..b0aefae41a1f 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/lmul.ll @@ -15,6 +15,7 @@ define void @load_store(ptr %p) { ; LMUL1-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64() ; LMUL1-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP1]] ; LMUL1-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; LMUL1-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() ; LMUL1-NEXT: br label [[VECTOR_BODY:%.*]] ; LMUL1: vector.body: ; LMUL1-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -24,7 +25,6 @@ define void @load_store(ptr %p) { ; LMUL1-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP4]], align 8 ; LMUL1-NEXT: [[TMP5:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; LMUL1-NEXT: store [[TMP5]], ptr [[TMP4]], align 8 -; LMUL1-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() ; LMUL1-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] ; LMUL1-NEXT: [[TMP7:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; LMUL1-NEXT: br i1 [[TMP7]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -57,6 +57,8 @@ define void @load_store(ptr %p) { ; LMUL2-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; LMUL2-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; LMUL2-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; LMUL2-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; LMUL2-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; LMUL2-NEXT: br label [[VECTOR_BODY:%.*]] ; LMUL2: vector.body: ; LMUL2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -66,8 +68,6 @@ define void @load_store(ptr %p) { ; LMUL2-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; LMUL2-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; LMUL2-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; LMUL2-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; LMUL2-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; LMUL2-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; LMUL2-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; LMUL2-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -100,6 +100,8 @@ define void @load_store(ptr %p) { ; LMUL4-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; LMUL4-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; LMUL4-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; LMUL4-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; LMUL4-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; LMUL4-NEXT: br label [[VECTOR_BODY:%.*]] ; LMUL4: vector.body: ; LMUL4-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -109,8 +111,6 @@ define void @load_store(ptr %p) { ; LMUL4-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; LMUL4-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; LMUL4-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; LMUL4-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; LMUL4-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; LMUL4-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; LMUL4-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; LMUL4-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -143,6 +143,8 @@ define void @load_store(ptr %p) { ; LMUL8-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 8 ; LMUL8-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; LMUL8-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; LMUL8-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; LMUL8-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8 ; LMUL8-NEXT: br label [[VECTOR_BODY:%.*]] ; LMUL8: vector.body: ; LMUL8-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -152,8 +154,6 @@ define void @load_store(ptr %p) { ; LMUL8-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; LMUL8-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; LMUL8-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; LMUL8-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; LMUL8-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 8 ; LMUL8-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; LMUL8-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; LMUL8-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll index ace267d72dea..0c5394cb95a6 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/low-trip-count.ll @@ -16,23 +16,23 @@ define void @trip5_i8(ptr noalias nocapture noundef %dst, ptr noalias nocapture ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 5, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 16 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP5]], i64 5) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i8, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr [[TMP7]], i32 1, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = shl [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i8, ptr [[TMP9]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr [[TMP10]], i32 1, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP11:%.*]] = add [[TMP8]], [[WIDE_MASKED_LOAD1]] -; CHECK-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP11]], ptr [[TMP10]], i32 1, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 16 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP13]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv16i1.i64(i64 [[TMP7]], i64 5) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i8, ptr [[SRC:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i8, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr [[TMP9]], i32 1, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP10:%.*]] = shl [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i8 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i8, ptr [[DST:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i8, ptr [[TMP11]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD1:%.*]] = call @llvm.masked.load.nxv16i8.p0(ptr [[TMP12]], i32 1, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP13:%.*]] = add [[TMP10]], [[WIDE_MASKED_LOAD1]] +; CHECK-NEXT: call void @llvm.masked.store.nxv16i8.p0( [[TMP13]], ptr [[TMP12]], i32 1, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll b/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll index 34b06972dab0..1307d57f32bc 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/mask-index-type.ll @@ -20,14 +20,16 @@ define void @test(ptr noalias nocapture %a, ptr noalias nocapture %b, i32 %v) { ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] -; VLENUNK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i64() -; VLENUNK-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer -; VLENUNK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) -; VLENUNK-NEXT: [[INDUCTION:%.*]] = add zeroinitializer, [[TMP6]] -; VLENUNK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 -; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 1, [[TMP8]] -; VLENUNK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP9]], i64 0 +; VLENUNK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 4 +; VLENUNK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv4i64() +; VLENUNK-NEXT: [[TMP7:%.*]] = add [[TMP6]], zeroinitializer +; VLENUNK-NEXT: [[TMP8:%.*]] = mul [[TMP7]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) +; VLENUNK-NEXT: [[INDUCTION:%.*]] = add zeroinitializer, [[TMP8]] +; VLENUNK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 +; VLENUNK-NEXT: [[TMP11:%.*]] = mul i64 1, [[TMP10]] +; VLENUNK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP11]], i64 0 ; VLENUNK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; VLENUNK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[V:%.*]], i64 0 ; VLENUNK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer @@ -35,20 +37,18 @@ define void @test(ptr noalias nocapture %a, ptr noalias nocapture %b, i32 %v) { ; VLENUNK: vector.body: ; VLENUNK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; VLENUNK-NEXT: [[VEC_IND:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; VLENUNK-NEXT: [[TMP10:%.*]] = add i64 [[INDEX]], 0 -; VLENUNK-NEXT: [[TMP11:%.*]] = icmp ult [[VEC_IND]], shufflevector ( insertelement ( poison, i64 512, i64 0), poison, zeroinitializer) -; VLENUNK-NEXT: [[TMP12:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[TMP10]] -; VLENUNK-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[TMP12]], i32 0 -; VLENUNK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP13]], i32 4, [[TMP11]], poison) -; VLENUNK-NEXT: [[TMP14:%.*]] = xor [[TMP11]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) -; VLENUNK-NEXT: [[PREDPHI:%.*]] = select [[TMP14]], zeroinitializer, [[WIDE_MASKED_LOAD]] -; VLENUNK-NEXT: [[TMP15:%.*]] = add [[PREDPHI]], [[BROADCAST_SPLAT]] -; VLENUNK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP10]] -; VLENUNK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[TMP16]], i32 0 -; VLENUNK-NEXT: store [[TMP15]], ptr [[TMP17]], align 4 -; VLENUNK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 -; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] +; VLENUNK-NEXT: [[TMP12:%.*]] = add i64 [[INDEX]], 0 +; VLENUNK-NEXT: [[TMP13:%.*]] = icmp ult [[VEC_IND]], shufflevector ( insertelement ( poison, i64 512, i64 0), poison, zeroinitializer) +; VLENUNK-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[A:%.*]], i64 [[TMP12]] +; VLENUNK-NEXT: [[TMP15:%.*]] = getelementptr i32, ptr [[TMP14]], i32 0 +; VLENUNK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP15]], i32 4, [[TMP13]], poison) +; VLENUNK-NEXT: [[TMP16:%.*]] = xor [[TMP13]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) +; VLENUNK-NEXT: [[PREDPHI:%.*]] = select [[TMP16]], zeroinitializer, [[WIDE_MASKED_LOAD]] +; VLENUNK-NEXT: [[TMP17:%.*]] = add [[PREDPHI]], [[BROADCAST_SPLAT]] +; VLENUNK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[TMP12]] +; VLENUNK-NEXT: [[TMP19:%.*]] = getelementptr inbounds i32, ptr [[TMP18]], i32 0 +; VLENUNK-NEXT: store [[TMP17]], ptr [[TMP19]], align 4 +; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; VLENUNK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; VLENUNK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP20]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll index 72e3bca79a5a..e50d7362365b 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/masked_gather_scatter.ll @@ -39,6 +39,8 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea ; RV32-NEXT: [[N_MOD_VF:%.*]] = urem i64 625, [[TMP4]] ; RV32-NEXT: [[N_VEC:%.*]] = sub i64 625, [[N_MOD_VF]] ; RV32-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 16 +; RV32-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() +; RV32-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 2 ; RV32-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; RV32-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer ; RV32-NEXT: [[TMP7:%.*]] = mul [[TMP6]], shufflevector ( insertelement ( poison, i64 16, i64 0), poison, zeroinitializer) @@ -62,8 +64,6 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea ; RV32-NEXT: [[TMP16:%.*]] = fadd [[WIDE_MASKED_GATHER6]], [[TMP15]] ; RV32-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[A]], [[VEC_IND]] ; RV32-NEXT: call void @llvm.masked.scatter.nxv2f64.nxv2p0( [[TMP16]], [[TMP17]], i32 8, [[TMP12]]), !alias.scope !5, !noalias !7 -; RV32-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; RV32-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 2 ; RV32-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] ; RV32-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; RV32-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -121,6 +121,8 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea ; RV64-NEXT: [[N_MOD_VF:%.*]] = urem i64 625, [[TMP4]] ; RV64-NEXT: [[N_VEC:%.*]] = sub i64 625, [[N_MOD_VF]] ; RV64-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 16 +; RV64-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() +; RV64-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 2 ; RV64-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; RV64-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer ; RV64-NEXT: [[TMP7:%.*]] = mul [[TMP6]], shufflevector ( insertelement ( poison, i64 16, i64 0), poison, zeroinitializer) @@ -144,8 +146,6 @@ define void @foo4(ptr nocapture %A, ptr nocapture readonly %B, ptr nocapture rea ; RV64-NEXT: [[TMP16:%.*]] = fadd [[WIDE_MASKED_GATHER6]], [[TMP15]] ; RV64-NEXT: [[TMP17:%.*]] = getelementptr inbounds double, ptr [[A]], [[VEC_IND]] ; RV64-NEXT: call void @llvm.masked.scatter.nxv2f64.nxv2p0( [[TMP16]], [[TMP17]], i32 8, [[TMP12]]), !alias.scope !5, !noalias !7 -; RV64-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; RV64-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 2 ; RV64-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] ; RV64-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; RV64-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll index cad64f5c7e2b..ca8c85712fc4 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/riscv-vector-reverse.ll @@ -51,10 +51,11 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1 ; CHECK-NEXT: LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: vp<%1> = original trip-count +; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count ; CHECK: ph: -; CHECK-NEXT: EMIT vp<%1> = EXPAND SCEV (zext i32 %n to i64) +; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64) ; CHECK-NEXT: No successors ; CHECK: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -70,7 +71,7 @@ define void @vector_reverse_i64(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: WIDEN ir<%add9> = add ir<%1>, ir<1> ; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> ; CHECK-NEXT: WIDEN store ir<%arrayidx3>, ir<%add9> -; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -188,10 +189,11 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: LV: Scalarizing: %cmp = icmp ugt i64 %indvars.iv, 1 ; CHECK-NEXT: LV: Scalarizing: %indvars.iv.next = add nsw i64 %indvars.iv, -1 ; CHECK-NEXT: VPlan 'Initial VPlan for VF={vscale x 4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count -; CHECK-NEXT: vp<%1> = original trip-count +; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count ; CHECK: ph: -; CHECK-NEXT: EMIT vp<%1> = EXPAND SCEV (zext i32 %n to i64) +; CHECK-NEXT: EMIT vp<[[TC]]> = EXPAND SCEV (zext i32 %n to i64) ; CHECK-NEXT: No successors ; CHECK: vector.ph: ; CHECK-NEXT: Successor(s): vector loop @@ -207,7 +209,7 @@ define void @vector_reverse_f32(ptr nocapture noundef writeonly %A, ptr nocaptur ; CHECK-NEXT: WIDEN ir<%conv1> = fadd ir<%1>, ir<1.000000e+00> ; CHECK-NEXT: CLONE ir<%arrayidx3> = getelementptr inbounds ir<%A>, ir<%idxprom> ; CHECK-NEXT: WIDEN store ir<%arrayidx3>, ir<%conv1> -; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll b/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll index 90c64e33bcef..77b0ae2f846a 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/safe-dep-distance.ll @@ -19,20 +19,20 @@ define void @test(ptr %p) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 32 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP4]], 200 -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i32 0 -; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP9]], align 32 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 32 +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP6]], 200 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP10]], i32 0 +; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP11]], align 32 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -142,20 +142,20 @@ define void @trivial_due_max_vscale(ptr %p) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 32 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP4]], 8192 -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i32 0 -; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP9]], align 32 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 32 +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP6]], 8192 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP10]], i32 0 +; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP11]], align 32 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: @@ -208,20 +208,20 @@ define void @no_high_lmul_or_interleave(ptr %p) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 200, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 200, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr i64, ptr [[TMP5]], i32 0 -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 32 -; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[TMP4]], 1024 -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP7]] -; CHECK-NEXT: [[TMP9:%.*]] = getelementptr i64, ptr [[TMP8]], i32 0 -; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP9]], align 32 -; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr i64, ptr [[P:%.*]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr i64, ptr [[TMP7]], i32 0 +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 32 +; CHECK-NEXT: [[TMP9:%.*]] = add i64 [[TMP6]], 1024 +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr i64, ptr [[P]], i64 [[TMP9]] +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr i64, ptr [[TMP10]], i32 0 +; CHECK-NEXT: store [[WIDE_LOAD]], ptr [[TMP11]], align 32 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll index f7bc4bd35f37..c634c2cf7fc6 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-basics.ll @@ -24,6 +24,8 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLENUNK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLENUNK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; VLENUNK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLENUNK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -35,8 +37,6 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLENUNK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; VLENUNK-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], [[BROADCAST_SPLAT]] ; VLENUNK-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; VLENUNK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; VLENUNK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -69,6 +69,8 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLEN128-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLEN128-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLEN128-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLEN128-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; VLEN128-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] @@ -80,8 +82,6 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLEN128-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; VLEN128-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], [[BROADCAST_SPLAT]] ; VLEN128-NEXT: store [[TMP7]], ptr [[TMP6]], align 8 -; VLEN128-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; VLEN128-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; VLEN128-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLEN128-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -135,6 +135,8 @@ define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) { ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLENUNK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; VLENUNK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[V:%.*]], i64 0 ; VLENUNK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLENUNK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -146,8 +148,6 @@ define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) { ; VLENUNK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; VLENUNK-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], [[BROADCAST_SPLAT]] ; VLENUNK-NEXT: store [[TMP7]], ptr [[TMP6]], align 4 -; VLENUNK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; VLENUNK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -180,6 +180,8 @@ define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) { ; VLEN128-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; VLEN128-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLEN128-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLEN128-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; VLEN128-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[V:%.*]], i64 0 ; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] @@ -191,8 +193,6 @@ define void @vector_add_i32(ptr noalias nocapture %a, i32 %v, i64 %n) { ; VLEN128-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; VLEN128-NEXT: [[TMP7:%.*]] = add [[WIDE_LOAD]], [[BROADCAST_SPLAT]] ; VLEN128-NEXT: store [[TMP7]], ptr [[TMP6]], align 4 -; VLEN128-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; VLEN128-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; VLEN128-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLEN128-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -301,6 +301,8 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLENUNK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLENUNK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; VLENUNK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLENUNK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -312,8 +314,6 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; VLENUNK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; VLENUNK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_LOAD]] ; VLENUNK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0( [[BROADCAST_SPLAT]], [[TMP7]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; VLENUNK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; VLENUNK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -346,6 +346,8 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; VLEN128-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLEN128-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLEN128-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLEN128-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; VLEN128-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] @@ -357,8 +359,6 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; VLEN128-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 ; VLEN128-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_LOAD]] ; VLEN128-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0( [[BROADCAST_SPLAT]], [[TMP7]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; VLEN128-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; VLEN128-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; VLEN128-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLEN128-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -409,6 +409,8 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLENUNK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; VLENUNK-NEXT: br label [[VECTOR_BODY:%.*]] ; VLENUNK: vector.body: ; VLENUNK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -420,8 +422,6 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 ; VLENUNK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_LOAD]] ; VLENUNK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv2i64.nxv2p0( [[TMP7]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; VLENUNK-NEXT: [[TMP8]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; VLENUNK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] ; VLENUNK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -459,6 +459,8 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 ; VLEN128-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLEN128-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLEN128-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLEN128-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; VLEN128-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] ; VLEN128: vector.body: ; VLEN128-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -470,8 +472,6 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 ; VLEN128-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_LOAD]] ; VLEN128-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv2i64.nxv2p0( [[TMP7]], i32 8, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; VLEN128-NEXT: [[TMP8]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; VLEN128-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; VLEN128-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] ; VLEN128-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLEN128-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -529,6 +529,8 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLENUNK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLENUNK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; VLENUNK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLENUNK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -538,8 +540,6 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLENUNK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; VLENUNK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 ; VLENUNK-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 -; VLENUNK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; VLENUNK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -570,6 +570,8 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLEN128-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLEN128-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLEN128-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLEN128-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; VLEN128-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] @@ -579,8 +581,6 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) { ; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; VLEN128-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 ; VLEN128-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 -; VLEN128-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; VLEN128-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; VLEN128-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLEN128-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -627,6 +627,8 @@ define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) { ; VLENUNK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLENUNK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLENUNK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLENUNK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; VLENUNK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLENUNK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[V:%.*]], i64 0 ; VLENUNK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLENUNK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -636,8 +638,6 @@ define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) { ; VLENUNK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; VLENUNK-NEXT: [[TMP6:%.*]] = getelementptr inbounds ptr, ptr [[TMP5]], i32 0 ; VLENUNK-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 -; VLENUNK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; VLENUNK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLENUNK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; VLENUNK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLENUNK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -668,6 +668,8 @@ define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) { ; VLEN128-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; VLEN128-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; VLEN128-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; VLEN128-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; VLEN128-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLEN128-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, ptr [[V:%.*]], i64 0 ; VLEN128-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; VLEN128-NEXT: br label [[VECTOR_BODY:%.*]] @@ -677,8 +679,6 @@ define void @splat_ptr(ptr noalias nocapture %a, ptr %v, i64 %n) { ; VLEN128-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; VLEN128-NEXT: [[TMP6:%.*]] = getelementptr inbounds ptr, ptr [[TMP5]], i32 0 ; VLEN128-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 -; VLEN128-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; VLEN128-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; VLEN128-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; VLEN128-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; VLEN128-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll index 0a8c7cfda9ef..affc6fc22c1b 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/scalable-tailfold.ll @@ -19,21 +19,21 @@ define void @vector_add(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = add [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP8]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP7]], i64 1025) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP10:%.*]] = add [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP10]], ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -85,21 +85,21 @@ define void @indexed_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_MASKED_LOAD]] -; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0( [[BROADCAST_SPLAT]], [[TMP8]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP7]], i64 1025) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_MASKED_LOAD]] +; CHECK-NEXT: call void @llvm.masked.scatter.nxv2i64.nxv2p0( [[BROADCAST_SPLAT]], [[TMP10]], i32 8, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: @@ -149,26 +149,26 @@ define i64 @indexed_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i64 ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP9:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_MASKED_LOAD]] -; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv2i64.nxv2p0( [[TMP8]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP9]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] -; CHECK-NEXT: [[TMP10:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP9]], [[VEC_PHI]] -; CHECK-NEXT: [[TMP11:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP12:%.*]] = mul i64 [[TMP11]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP12]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP11:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP7]], i64 1025) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], [[WIDE_MASKED_LOAD]] +; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv2i64.nxv2p0( [[TMP10]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP11]] = add [[VEC_PHI]], [[WIDE_MASKED_GATHER]] +; CHECK-NEXT: [[TMP12:%.*]] = select [[ACTIVE_LANE_MASK]], [[TMP11]], [[VEC_PHI]] +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[TMP13:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP13]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64( [[TMP10]]) +; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vector.reduce.add.nxv2i64( [[TMP12]]) ; CHECK-NEXT: br i1 true, label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] @@ -221,19 +221,19 @@ define void @splat_int(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP7]], i64 1025) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] ; CHECK: middle.block: @@ -279,20 +279,20 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1025) +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP7]], i64 1025) ; CHECK-NEXT: store i64 [[V]], ptr [[B:%.*]], align 8 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: middle.block: @@ -373,21 +373,21 @@ define void @vector_add_trip1024(ptr noalias nocapture %a, i64 %v, i64 %n) { ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i64 1024, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = mul i64 [[TMP5]], 2 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP5:%.*]] = add i64 [[INDEX]], 0 -; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP5]], i64 1024) -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 -; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]], poison) -; CHECK-NEXT: [[TMP8:%.*]] = add [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP8]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[TMP7:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[ACTIVE_LANE_MASK:%.*]] = call @llvm.get.active.lane.mask.nxv2i1.i64(i64 [[TMP7]], i64 1024) +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP7]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i32 0 +; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv2i64.p0(ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]], poison) +; CHECK-NEXT: [[TMP10:%.*]] = add [[WIDE_MASKED_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[TMP10]], ptr [[TMP9]], i32 8, [[ACTIVE_LANE_MASK]]) +; CHECK-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll b/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll index c7bb4ca4b6c6..34a7987bb40a 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/select-cmp-reduction.ll @@ -38,6 +38,8 @@ define i32 @select_icmp(i32 %x, i32 %y, ptr nocapture readonly %c, i64 %n) #0 { ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[X:%.*]], i64 0 ; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[Y:%.*]], i64 0 @@ -52,8 +54,6 @@ define i32 @select_icmp(i32 %x, i32 %y, ptr nocapture readonly %c, i64 %n) #0 { ; SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; SCALABLE-NEXT: [[TMP8:%.*]] = icmp slt [[WIDE_LOAD]], [[BROADCAST_SPLAT]] ; SCALABLE-NEXT: [[TMP9]] = select [[TMP8]], [[VEC_PHI]], [[BROADCAST_SPLAT2]] -; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -113,6 +113,8 @@ define i32 @select_fcmp(float %x, i32 %y, ptr nocapture readonly %c, i64 %n) #0 ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, float [[X:%.*]], i64 0 ; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement poison, i32 [[Y:%.*]], i64 0 @@ -127,8 +129,6 @@ define i32 @select_fcmp(float %x, i32 %y, ptr nocapture readonly %c, i64 %n) #0 ; SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; SCALABLE-NEXT: [[TMP8:%.*]] = fcmp fast olt [[WIDE_LOAD]], [[BROADCAST_SPLAT]] ; SCALABLE-NEXT: [[TMP9]] = select [[TMP8]], [[VEC_PHI]], [[BROADCAST_SPLAT2]] -; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -184,6 +184,8 @@ define i32 @select_const_i32_from_icmp(ptr nocapture readonly %v, i64 %n) #0 { ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALABLE: vector.body: ; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -194,8 +196,6 @@ define i32 @select_const_i32_from_icmp(ptr nocapture readonly %v, i64 %n) #0 { ; SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; SCALABLE-NEXT: [[TMP8:%.*]] = icmp eq [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 3, i64 0), poison, zeroinitializer) ; SCALABLE-NEXT: [[TMP9]] = select [[TMP8]], [[VEC_PHI]], shufflevector ( insertelement ( poison, i32 7, i64 0), poison, zeroinitializer) -; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -257,6 +257,8 @@ define i32 @select_i32_from_icmp(ptr nocapture readonly %v, i32 %a, i32 %b, i64 ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[MINMAX_IDENT_SPLATINSERT:%.*]] = insertelement poison, i32 [[A:%.*]], i64 0 ; SCALABLE-NEXT: [[MINMAX_IDENT_SPLAT:%.*]] = shufflevector [[MINMAX_IDENT_SPLATINSERT]], poison, zeroinitializer ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[B:%.*]], i64 0 @@ -271,8 +273,6 @@ define i32 @select_i32_from_icmp(ptr nocapture readonly %v, i32 %a, i32 %b, i64 ; SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; SCALABLE-NEXT: [[TMP8:%.*]] = icmp eq [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 3, i64 0), poison, zeroinitializer) ; SCALABLE-NEXT: [[TMP9]] = select [[TMP8]], [[VEC_PHI]], [[BROADCAST_SPLAT]] -; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -330,6 +330,8 @@ define i32 @select_const_i32_from_fcmp(ptr nocapture readonly %v, i64 %n) #0 { ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALABLE: vector.body: ; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -340,8 +342,6 @@ define i32 @select_const_i32_from_fcmp(ptr nocapture readonly %v, i64 %n) #0 { ; SCALABLE-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 4 ; SCALABLE-NEXT: [[TMP8:%.*]] = fcmp fast ueq [[WIDE_LOAD]], shufflevector ( insertelement ( poison, float 3.000000e+00, i64 0), poison, zeroinitializer) ; SCALABLE-NEXT: [[TMP9]] = select [[TMP8]], [[VEC_PHI]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) -; SCALABLE-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP11:%.*]] = mul i64 [[TMP10]], 4 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP11]] ; SCALABLE-NEXT: [[TMP12:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP12]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -428,6 +428,8 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1 ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 %n, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 %n, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALABLE: vector.body: ; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH:%.*]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -444,8 +446,6 @@ define i32 @pred_select_const_i32_from_icmp(ptr noalias nocapture readonly %src1 ; SCALABLE-NEXT: [[TMP13:%.*]] = select [[TMP12]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer), [[VEC_PHI]] ; SCALABLE-NEXT: [[TMP14:%.*]] = xor [[TMP8]], shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer) ; SCALABLE-NEXT: [[PREDPHI]] = select [[TMP8]], [[TMP13]], [[VEC_PHI]] -; SCALABLE-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; SCALABLE-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll b/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll index 38ef0537e431..331a8d2eb66e 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/short-trip-count.ll @@ -14,6 +14,8 @@ define void @small_trip_count_min_vlen_128(ptr nocapture %a) nounwind vscale_ran ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 4, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -24,8 +26,6 @@ define void @small_trip_count_min_vlen_128(ptr nocapture %a) nounwind vscale_ran ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP7]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP8:%.*]] = add nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP8]], ptr [[TMP7]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP10]] ; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -75,6 +75,8 @@ define void @small_trip_count_min_vlen_32(ptr nocapture %a) nounwind vscale_rang ; CHECK-NEXT: [[N_RND_UP:%.*]] = add i32 4, [[TMP4]] ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N_RND_UP]], [[TMP1]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N_RND_UP]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -85,8 +87,6 @@ define void @small_trip_count_min_vlen_32(ptr nocapture %a) nounwind vscale_rang ; CHECK-NEXT: [[WIDE_MASKED_LOAD:%.*]] = call @llvm.masked.load.nxv4i32.p0(ptr [[TMP7]], i32 4, [[ACTIVE_LANE_MASK]], poison) ; CHECK-NEXT: [[TMP8:%.*]] = add nsw [[WIDE_MASKED_LOAD]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: call void @llvm.masked.store.nxv4i32.p0( [[TMP8]], ptr [[TMP7]], i32 4, [[ACTIVE_LANE_MASK]]) -; CHECK-NEXT: [[TMP9:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP10:%.*]] = mul i32 [[TMP9]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add i32 [[INDEX]], [[TMP10]] ; CHECK-NEXT: br i1 true, label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll index 65a2cb51b88f..12fdf2149daf 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/strided-accesses.ll @@ -17,6 +17,8 @@ define void @single_constant_stride_int_scaled(ptr %p) { ; CHECK-NEXT: [[TMP4:%.*]] = icmp eq i64 [[N_MOD_VF]], 0 ; CHECK-NEXT: [[TMP5:%.*]] = select i1 [[TMP4]], i64 [[TMP3]], i64 [[N_MOD_VF]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[TMP5]] +; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP7:%.*]] = add [[TMP6]], zeroinitializer ; CHECK-NEXT: [[TMP8:%.*]] = mul [[TMP7]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -35,8 +37,6 @@ define void @single_constant_stride_int_scaled(ptr %p) { ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[TMP13]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; CHECK-NEXT: [[TMP14:%.*]] = add [[WIDE_MASKED_GATHER]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP14]], [[TMP13]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -90,6 +90,8 @@ define void @single_constant_stride_int_iv(ptr %p) { ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] ; CHECK-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], 64 +; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 64, i64 0), poison, zeroinitializer) @@ -107,8 +109,6 @@ define void @single_constant_stride_int_iv(ptr %p) { ; CHECK-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[TMP10]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison) ; CHECK-NEXT: [[TMP11:%.*]] = add [[WIDE_MASKED_GATHER]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP11]], [[TMP10]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = mul i64 [[TMP12]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP13]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP14:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -236,6 +236,8 @@ define void @single_stride_int_scaled(ptr %p, i64 %stride) { ; NOSTRIDED-NEXT: [[TMP4:%.*]] = mul i64 [[TMP3]], 4 ; NOSTRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]] ; NOSTRIDED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 ; NOSTRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; NOSTRIDED: vector.body: ; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -245,8 +247,6 @@ define void @single_stride_int_scaled(ptr %p, i64 %stride) { ; NOSTRIDED-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 4 ; NOSTRIDED-NEXT: [[TMP8:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; NOSTRIDED-NEXT: store [[TMP8]], ptr [[TMP7]], align 4 -; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 ; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] ; NOSTRIDED-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; NOSTRIDED-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -320,6 +320,8 @@ define void @single_stride_int_iv(ptr %p, i64 %stride) { ; NOSTRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]] ; NOSTRIDED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] ; NOSTRIDED-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], [[STRIDE]] +; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 ; NOSTRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; NOSTRIDED: vector.body: ; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -329,8 +331,6 @@ define void @single_stride_int_iv(ptr %p, i64 %stride) { ; NOSTRIDED-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 4 ; NOSTRIDED-NEXT: [[TMP8:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; NOSTRIDED-NEXT: store [[TMP8]], ptr [[TMP7]], align 4 -; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 ; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] ; NOSTRIDED-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; NOSTRIDED-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -452,6 +452,8 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) { ; NOSTRIDED-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 4 ; NOSTRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP8]] ; NOSTRIDED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; NOSTRIDED-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; NOSTRIDED-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; NOSTRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; NOSTRIDED: vector.body: ; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -463,8 +465,6 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) { ; NOSTRIDED-NEXT: [[TMP13:%.*]] = getelementptr i32, ptr [[P2]], i64 [[TMP9]] ; NOSTRIDED-NEXT: [[TMP14:%.*]] = getelementptr i32, ptr [[TMP13]], i32 0 ; NOSTRIDED-NEXT: store [[TMP12]], ptr [[TMP14]], align 4 -; NOSTRIDED-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; NOSTRIDED-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 4 ; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; NOSTRIDED-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; NOSTRIDED-NEXT: br i1 [[TMP17]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -518,6 +518,8 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) { ; STRIDED-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; STRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP9]] ; STRIDED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; STRIDED-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() +; STRIDED-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; STRIDED-NEXT: [[TMP10:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; STRIDED-NEXT: [[TMP11:%.*]] = add [[TMP10]], zeroinitializer ; STRIDED-NEXT: [[TMP12:%.*]] = mul [[TMP11]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -539,8 +541,6 @@ define void @double_stride_int_scaled(ptr %p, ptr %p2, i64 %stride) { ; STRIDED-NEXT: [[TMP18:%.*]] = add [[WIDE_MASKED_GATHER]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; STRIDED-NEXT: [[TMP19:%.*]] = getelementptr i32, ptr [[P2]], [[TMP16]] ; STRIDED-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP18]], [[TMP19]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)), !alias.scope !11, !noalias !8 -; STRIDED-NEXT: [[TMP20:%.*]] = call i64 @llvm.vscale.i64() -; STRIDED-NEXT: [[TMP21:%.*]] = mul i64 [[TMP20]], 4 ; STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP21]] ; STRIDED-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; STRIDED-NEXT: [[TMP22:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -601,6 +601,8 @@ define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) { ; NOSTRIDED-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP4]] ; NOSTRIDED-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] ; NOSTRIDED-NEXT: [[IND_END:%.*]] = mul i64 [[N_VEC]], [[STRIDE]] +; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 ; NOSTRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; NOSTRIDED: vector.body: ; NOSTRIDED-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -610,8 +612,6 @@ define void @double_stride_int_iv(ptr %p, ptr %p2, i64 %stride) { ; NOSTRIDED-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 4 ; NOSTRIDED-NEXT: [[TMP8:%.*]] = add [[WIDE_LOAD]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; NOSTRIDED-NEXT: store [[TMP8]], ptr [[TMP7]], align 4 -; NOSTRIDED-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; NOSTRIDED-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 4 ; NOSTRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] ; NOSTRIDED-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; NOSTRIDED-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] @@ -729,6 +729,8 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) { ; STRIDED-NEXT: [[IND_END:%.*]] = getelementptr i8, ptr [[P]], i64 [[TMP10]] ; STRIDED-NEXT: [[TMP11:%.*]] = mul i64 [[N_VEC]], [[STRIDE]] ; STRIDED-NEXT: [[IND_END7:%.*]] = getelementptr i8, ptr [[P2]], i64 [[TMP11]] +; STRIDED-NEXT: [[TMP29:%.*]] = call i64 @llvm.vscale.i64() +; STRIDED-NEXT: [[TMP30:%.*]] = mul i64 [[TMP29]], 4 ; STRIDED-NEXT: br label [[VECTOR_BODY:%.*]] ; STRIDED: vector.body: ; STRIDED-NEXT: [[POINTER_PHI:%.*]] = phi ptr [ [[P]], [[VECTOR_PH]] ], [ [[PTR_IND:%.*]], [[VECTOR_BODY]] ] @@ -761,8 +763,6 @@ define void @double_stride_ptr_iv(ptr %p, ptr %p2, i64 %stride) { ; STRIDED-NEXT: [[WIDE_MASKED_GATHER:%.*]] = call @llvm.masked.gather.nxv4i32.nxv4p0( [[TMP19]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer), poison), !alias.scope !15 ; STRIDED-NEXT: [[TMP28:%.*]] = add [[WIDE_MASKED_GATHER]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) ; STRIDED-NEXT: call void @llvm.masked.scatter.nxv4i32.nxv4p0( [[TMP28]], [[TMP27]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)), !alias.scope !18, !noalias !15 -; STRIDED-NEXT: [[TMP29:%.*]] = call i64 @llvm.vscale.i64() -; STRIDED-NEXT: [[TMP30:%.*]] = mul i64 [[TMP29]], 4 ; STRIDED-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP30]] ; STRIDED-NEXT: [[PTR_IND]] = getelementptr i8, ptr [[POINTER_PHI]], i64 [[TMP15]] ; STRIDED-NEXT: [[PTR_IND12]] = getelementptr i8, ptr [[POINTER_PHI11]], i64 [[TMP23]] diff --git a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll index 89c3219c4b6d..a12dfbaec195 100644 --- a/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll +++ b/llvm/test/Transforms/LoopVectorize/RISCV/uniform-load-store.ll @@ -21,6 +21,8 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6 ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALABLE: vector.body: ; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -31,8 +33,6 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6 ; SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP7]], align 8 -; SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -102,6 +102,8 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6 ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; TF-SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; TF-SCALABLE: vector.body: ; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -113,8 +115,6 @@ define void @uniform_load(ptr noalias nocapture %a, ptr noalias nocapture %b, i6 ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] ; TF-SCALABLE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP8]], i32 8, [[ACTIVE_LANE_MASK]]) -; TF-SCALABLE-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]] ; TF-SCALABLE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TF-SCALABLE-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -196,6 +196,8 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALABLE: vector.body: ; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -206,8 +208,6 @@ define i64 @uniform_load_outside_use(ptr noalias nocapture %a, ptr noalias nocap ; SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP7]], align 8 -; SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] @@ -326,6 +326,8 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 ; SCALABLE-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; SCALABLE-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; SCALABLE-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -349,8 +351,6 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca ; SCALABLE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP10]] ; SCALABLE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i32 0 ; SCALABLE-NEXT: store [[PREDPHI]], ptr [[TMP14]], align 8 -; SCALABLE-NEXT: [[TMP15:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP16:%.*]] = mul i64 [[TMP15]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP16]] ; SCALABLE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; SCALABLE-NEXT: [[TMP17:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -443,6 +443,8 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 ; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; TF-SCALABLE-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = mul [[TMP6]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -470,8 +472,6 @@ define void @conditional_uniform_load(ptr noalias nocapture %a, ptr noalias noca ; TF-SCALABLE-NEXT: [[TMP17:%.*]] = or [[TMP13]], [[TMP15]] ; TF-SCALABLE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP16]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[PREDPHI]], ptr [[TMP18]], i32 8, [[TMP17]]) -; TF-SCALABLE-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP20]] ; TF-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; TF-SCALABLE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -581,6 +581,8 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; SCALABLE: vector.body: ; SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -591,8 +593,6 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt ; SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP7]], align 8 -; SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -662,6 +662,8 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; TF-SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] ; TF-SCALABLE: vector.body: ; TF-SCALABLE-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] @@ -673,8 +675,6 @@ define void @uniform_load_unaligned(ptr noalias nocapture %a, ptr noalias nocapt ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] ; TF-SCALABLE-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP8]], i32 8, [[ACTIVE_LANE_MASK]]) -; TF-SCALABLE-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP10:%.*]] = mul i64 [[TMP9]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP10]] ; TF-SCALABLE-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TF-SCALABLE-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] @@ -756,6 +756,8 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -766,8 +768,6 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; SCALABLE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 -; SCALABLE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; SCALABLE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] @@ -837,6 +837,8 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; TF-SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -848,8 +850,6 @@ define void @uniform_store(ptr noalias nocapture %a, ptr noalias nocapture %b, i ; TF-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; TF-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] ; TF-SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TF-SCALABLE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP8:![0-9]+]] @@ -931,6 +931,8 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 2 ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -952,8 +954,6 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias ; SCALABLE-NEXT: [[TMP14:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP8]] ; SCALABLE-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP14]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP15]], align 8 -; SCALABLE-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP17:%.*]] = mul i64 [[TMP16]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP17]] ; SCALABLE-NEXT: [[TMP18:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP18]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] @@ -1029,6 +1029,8 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2 ; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; TF-SCALABLE-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = mul [[TMP6]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -1052,8 +1054,6 @@ define void @uniform_store_of_loop_varying(ptr noalias nocapture %a, ptr noalias ; TF-SCALABLE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP11]] ; TF-SCALABLE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT2]], ptr [[TMP13]], i32 8, [[ACTIVE_LANE_MASK]]) -; TF-SCALABLE-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP15]] ; TF-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; TF-SCALABLE-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1162,6 +1162,8 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2 ; SCALABLE-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; SCALABLE-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; SCALABLE-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -1185,8 +1187,6 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc ; SCALABLE-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP10]] ; SCALABLE-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP12]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP13]], align 8 -; SCALABLE-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP15:%.*]] = mul i64 [[TMP14]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] ; SCALABLE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; SCALABLE-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1275,6 +1275,8 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 ; TF-SCALABLE-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv2i64() ; TF-SCALABLE-NEXT: [[TMP6:%.*]] = add [[TMP5]], zeroinitializer ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = mul [[TMP6]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -1303,8 +1305,6 @@ define void @conditional_uniform_store(ptr noalias nocapture %a, ptr noalias noc ; TF-SCALABLE-NEXT: [[TMP17:%.*]] = or [[TMP13]], [[TMP16]] ; TF-SCALABLE-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP14]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP18]], i32 8, [[TMP17]]) -; TF-SCALABLE-NEXT: [[TMP19:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP20:%.*]] = mul i64 [[TMP19]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP20]] ; TF-SCALABLE-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; TF-SCALABLE-NEXT: [[TMP21:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] @@ -1412,6 +1412,8 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap ; SCALABLE-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 1025, [[TMP3]] ; SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 1025, [[N_MOD_VF]] +; SCALABLE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; SCALABLE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -1422,8 +1424,6 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap ; SCALABLE-NEXT: [[TMP5:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP4]] ; SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[TMP5]], i32 0 ; SCALABLE-NEXT: store [[BROADCAST_SPLAT]], ptr [[TMP6]], align 8 -; SCALABLE-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; SCALABLE-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 ; SCALABLE-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] ; SCALABLE-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; SCALABLE-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] @@ -1493,6 +1493,8 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap ; TF-SCALABLE-NEXT: [[N_RND_UP:%.*]] = add i64 1025, [[TMP4]] ; TF-SCALABLE-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N_RND_UP]], [[TMP1]] ; TF-SCALABLE-NEXT: [[N_VEC:%.*]] = sub i64 [[N_RND_UP]], [[N_MOD_VF]] +; TF-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; TF-SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TF-SCALABLE-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i64 [[V:%.*]], i64 0 ; TF-SCALABLE-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; TF-SCALABLE-NEXT: br label [[VECTOR_BODY:%.*]] @@ -1504,8 +1506,6 @@ define void @uniform_store_unaligned(ptr noalias nocapture %a, ptr noalias nocap ; TF-SCALABLE-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[TMP5]] ; TF-SCALABLE-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i32 0 ; TF-SCALABLE-NEXT: call void @llvm.masked.store.nxv2i64.p0( [[BROADCAST_SPLAT]], ptr [[TMP7]], i32 8, [[ACTIVE_LANE_MASK]]) -; TF-SCALABLE-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; TF-SCALABLE-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 2 ; TF-SCALABLE-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], [[TMP9]] ; TF-SCALABLE-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; TF-SCALABLE-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll index 8ff542a36732..d93cf2b14f8a 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-chains-vplan.ll @@ -5,6 +5,7 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-LABEL: 'test_chained_first_order_recurrences_1' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: @@ -23,7 +24,7 @@ define void @test_chained_first_order_recurrences_1(ptr %ptr) { ; CHECK-NEXT: EMIT vp<[[FOR2_SPLICE:%.+]]> = first-order splice ir<%for.2>, vp<[[FOR1_SPLICE]]> ; CHECK-NEXT: WIDEN ir<%add> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> ; CHECK-NEXT: WIDEN store ir<%gep.ptr>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -55,6 +56,7 @@ exit: define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-LABEL: 'test_chained_first_order_recurrences_3' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: @@ -76,7 +78,7 @@ define void @test_chained_first_order_recurrences_3(ptr %ptr) { ; CHECK-NEXT: WIDEN ir<%add.1> = add vp<[[FOR1_SPLICE]]>, vp<[[FOR2_SPLICE]]> ; CHECK-NEXT: WIDEN ir<%add.2> = add ir<%add.1>, vp<[[FOR3_SPLICE]]> ; CHECK-NEXT: WIDEN store ir<%gep.ptr>, ir<%add.2> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll index 3198f19a8f9a..060d28f030f7 100644 --- a/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll +++ b/llvm/test/Transforms/LoopVectorize/first-order-recurrence-sink-replicate-region.ll @@ -9,6 +9,7 @@ target datalayout = "e-m:e-i64:64-i128:128-n32:64-S128" define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize { ; CHECK-LABEL: sink_replicate_region_1 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<20001> = original trip-count @@ -65,7 +66,7 @@ define void @sink_replicate_region_1(i32 %x, ptr %ptr, ptr noalias %dst) optsize ; CHECK-NEXT: Successor(s): loop.2 ; CHECK-EMPTY: ; CHECK-NEXT: loop.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -99,6 +100,7 @@ exit: define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-LABEL: sink_replicate_region_2 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<20001> = original trip-count @@ -136,7 +138,7 @@ define void @sink_replicate_region_2(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: Successor(s): loop.1 ; CHECK-EMPTY: ; CHECK-NEXT: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -168,6 +170,7 @@ exit: define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-LABEL: sink_replicate_region_3_reduction ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<20001> = original trip-count @@ -205,7 +208,7 @@ define i32 @sink_replicate_region_3_reduction(i32 %x, i8 %y, ptr %ptr) optsize { ; CHECK-NEXT: WIDEN ir<%add> = add vp<[[PRED]]>, ir<%recur.next> ; CHECK-NEXT: WIDEN ir<%and.red.next> = and ir<%and.red>, ir<%add> ; CHECK-NEXT: EMIT vp<[[SEL:%.+]]> = select vp<[[MASK]]>, ir<%and.red.next>, ir<%and.red> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -242,6 +245,7 @@ exit: define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr %ptr, ptr noalias %dst) optsize { ; CHECK-LABEL: sink_replicate_region_4_requires_split_at_end_of_block ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<20001> = original trip-count @@ -302,7 +306,7 @@ define void @sink_replicate_region_4_requires_split_at_end_of_block(i32 %x, ptr ; CHECK-NEXT: Successor(s): loop.3 ; CHECK-EMPTY: ; CHECK: loop.3: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -340,6 +344,7 @@ exit: define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias %dst.2, i32 %x, i8 %y) optsize { ; CHECK-LABEL: sink_replicate_region_after_replicate_region ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -384,7 +389,7 @@ define void @sink_replicate_region_after_replicate_region(ptr %ptr, ptr noalias ; CHECK-NEXT: Successor(s): loop.3 ; CHECK-EMPTY: ; CHECK-NEXT: loop.3: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -418,6 +423,7 @@ exit: ; preds = %loop define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias %dst) { ; CHECK-LABEL: need_new_block_after_sinking_pr56146 ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<3> = original trip-count @@ -455,7 +461,7 @@ define void @need_new_block_after_sinking_pr56146(i32 %x, ptr %src, ptr noalias ; CHECK-NEXT: Successor(s): loop.1 ; CHECK-EMPTY: ; CHECK-NEXT: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll index 6dac3531277d..b4ab6f7e8ceb 100644 --- a/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll +++ b/llvm/test/Transforms/LoopVectorize/icmp-uniforms.ll @@ -37,6 +37,7 @@ for.end: ; Check for crash exposed by D76992. ; CHECK-LABEL: 'test' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<14> = original trip-count @@ -70,7 +71,7 @@ for.end: ; CHECK-NEXT: Successor(s): loop.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successor ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll index 028e0b7b39cb..7b9d7f798639 100644 --- a/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll +++ b/llvm/test/Transforms/LoopVectorize/interleave-and-scalarize-only.ll @@ -5,6 +5,7 @@ ; DBG-LABEL: 'test_scalarize_call' ; DBG: VPlan 'Initial VPlan for VF={1},UF>=1' { +; DBG-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; DBG-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; DBG-NEXT: vp<[[TC:%.+]]> = original trip-count ; DBG-EMPTY: @@ -23,7 +24,7 @@ ; DBG-NEXT: CLONE ir<%min> = call @llvm.smin.i32(vp<[[IV_STEPS]]>, ir<65535>) ; DBG-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%dst>, vp<[[IV_STEPS]]> ; DBG-NEXT: CLONE store ir<%min>, ir<%arrayidx> -; DBG-NEXT: EMIT vp<[[INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; DBG-NEXT: EMIT vp<[[INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; DBG-NEXT: EMIT branch-on-count vp<[[INC]]>, vp<[[VEC_TC]]> ; DBG-NEXT: No successors ; DBG-NEXT: } @@ -67,7 +68,8 @@ declare i32 @llvm.smin.i32(i32, i32) ; DBG-LABEL: 'test_scalarize_with_branch_cond' -; DBG: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count +; DBG: Live-in vp<[[VFxUF:%.+]]> = VF * UF +; DBG-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; DBG-NEXT: Live-in ir<1000> = original trip-count ; DBG-EMPTY: ; DBG-NEXT: vector.ph: @@ -100,7 +102,7 @@ declare i32 @llvm.smin.i32(i32, i32) ; DBG-NEXT: Successor(s): cond.false.1 ; DBG-EMPTY: ; DBG-NEXT: cond.false.1: -; DBG-NEXT: EMIT vp<[[CAN_IV_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; DBG-NEXT: EMIT vp<[[CAN_IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; DBG-NEXT: EMIT branch-on-count vp<[[CAN_IV_INC]]>, vp<[[VEC_TC]]> ; DBG-NEXT: No successors ; DBG-NEXT: } @@ -173,6 +175,7 @@ exit: ; DBG-LABEL: 'first_order_recurrence_using_induction' ; DBG: VPlan 'Initial VPlan for VF={1},UF>=1' { +; DBG-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; DBG-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; DBG-NEXT: vp<[[TC:%.+]]> = original trip-count ; DBG-EMPTY: @@ -191,7 +194,7 @@ exit: ; DBG-NEXT: vp<[[SCALAR_STEPS]]> = SCALAR-STEPS vp<[[DERIVED_IV]]>, ir<1> ; DBG-NEXT: EMIT vp<[[SPLICE:%.+]]> = first-order splice ir<%for>, vp<[[SCALAR_STEPS]]> ; DBG-NEXT: CLONE store vp<[[SPLICE]]>, ir<%dst> -; DBG-NEXT: EMIT vp<[[IV_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; DBG-NEXT: EMIT vp<[[IV_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; DBG-NEXT: EMIT branch-on-count vp<[[IV_INC]]>, vp<[[VTC]]> ; DBG-NEXT: No successors ; DBG-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll index c7d7ac3cb9ed..9023916d6e1c 100644 --- a/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll +++ b/llvm/test/Transforms/LoopVectorize/outer_loop_scalable.ll @@ -24,6 +24,8 @@ define void @foo() { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 1024, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 1024, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 ; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i64() ; CHECK-NEXT: [[TMP5:%.*]] = add [[TMP4]], zeroinitializer ; CHECK-NEXT: [[TMP6:%.*]] = mul [[TMP5]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) @@ -55,8 +57,6 @@ define void @foo() { ; CHECK-NEXT: call void @llvm.masked.scatter.nxv4f32.nxv4p0( [[VEC_PHI5]], [[TMP10]], i32 4, shufflevector ( insertelement ( poison, i1 true, i64 0), poison, zeroinitializer)) ; CHECK-NEXT: [[TMP16:%.*]] = add nuw nsw [[VEC_IND]], shufflevector ( insertelement ( poison, i64 1, i64 0), poison, zeroinitializer) ; CHECK-NEXT: [[TMP17:%.*]] = icmp eq [[TMP16]], shufflevector ( insertelement ( poison, i64 1024, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP18:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP19:%.*]] = mul i64 [[TMP18]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP19]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP20:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] diff --git a/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll b/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll index 69fbb6519f6a..3ba57821bc31 100644 --- a/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll +++ b/llvm/test/Transforms/LoopVectorize/reduction-inloop-cond.ll @@ -1,89 +1,91 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals none --version 4 ; RUN: opt < %s -passes=loop-vectorize,dce,instcombine -force-vector-interleave=1 -force-vector-width=4 -prefer-inloop-reductions -S | FileCheck %s define float @cond_fadd(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %cond, i64 %N){ -; CHECK-LABEL: @cond_fadd( +; CHECK-LABEL: define float @cond_fadd( +; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[COND:%.*]], i64 [[N:%.*]]) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[PRED_LOAD_CONTINUE6]] ] -; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP26:%.*]], [[PRED_LOAD_CONTINUE6]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = fcmp une <4 x float> [[WIDE_LOAD]], -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0 -; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = fcmp une <4 x float> [[WIDE_LOAD]], +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 +; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] ; CHECK: pred.load.if: -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[TMP5:%.*]] = load float, ptr [[TMP4]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> poison, float [[TMP4]], i64 0 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] ; CHECK: pred.load.continue: -; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ] -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1 -; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] +; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 +; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] ; CHECK: pred.load.if1: -; CHECK-NEXT: [[TMP9:%.*]] = or disjoint i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]] -; CHECK-NEXT: [[TMP11:%.*]] = load float, ptr [[TMP10]], align 4 -; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP11]], i64 1 +; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x float> [[TMP6]], float [[TMP10]], i64 1 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]] ; CHECK: pred.load.continue2: -; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x float> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ] -; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2 -; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] +; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x float> [ [[TMP6]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], [[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 +; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] ; CHECK: pred.load.if3: -; CHECK-NEXT: [[TMP15:%.*]] = or disjoint i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[TMP16]], align 4 -; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> [[TMP13]], float [[TMP17]], i64 2 +; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4 +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x float> [[TMP12]], float [[TMP16]], i64 2 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]] ; CHECK: pred.load.continue4: -; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x float> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ] -; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3 -; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] +; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x float> [ [[TMP12]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP17]], [[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 +; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.if5: -; CHECK-NEXT: [[TMP21:%.*]] = or disjoint i64 [[INDEX]], 3 -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = load float, ptr [[TMP22]], align 4 -; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x float> [[TMP19]], float [[TMP23]], i64 3 +; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP20]] +; CHECK-NEXT: [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x float> [[TMP18]], float [[TMP22]], i64 3 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.continue6: -; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x float> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ] -; CHECK-NEXT: [[TMP26:%.*]] = select fast <4 x i1> [[TMP2]], <4 x float> [[TMP25]], <4 x float> zeroinitializer -; CHECK-NEXT: [[TMP27]] = call fast float @llvm.vector.reduce.fadd.v4f32(float [[VEC_PHI]], <4 x float> [[TMP26]]) +; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x float> [ [[TMP18]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP25:%.*]] = select fast <4 x i1> [[TMP1]], <4 x float> [[TMP24]], <4 x float> zeroinitializer +; CHECK-NEXT: [[TMP26]] = call fast float @llvm.vector.reduce.fadd.v4f32(float [[VEC_PHI]], <4 x float> [[TMP25]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] +; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP27]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP26]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ] ; CHECK-NEXT: [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[IV]] -; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[ARRAYIDX]], align 4 -; CHECK-NEXT: [[TOBOOL:%.*]] = fcmp une float [[TMP29]], 5.000000e+00 +; CHECK-NEXT: [[TMP28:%.*]] = load float, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[TOBOOL:%.*]] = fcmp une float [[TMP28]], 5.000000e+00 ; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]] ; CHECK: if.then: ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] -; CHECK-NEXT: [[TMP30:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 -; CHECK-NEXT: [[FADD:%.*]] = fadd fast float [[RDX]], [[TMP30]] +; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[FADD:%.*]] = fadd fast float [[RDX]], [[TMP29]] ; CHECK-NEXT: br label [[FOR_INC]] ; CHECK: for.inc: ; CHECK-NEXT: [[RES]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[FADD]], [[IF_THEN]] ] ; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] -; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP2:![0-9]+]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP3:![0-9]+]] ; CHECK: for.end: -; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP26]], [[MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret float [[RES_LCSSA]] ; entry: @@ -114,90 +116,91 @@ for.end: } define float @cond_cmp_sel(ptr noalias %a, ptr noalias %cond, i64 %N) { -; CHECK-LABEL: @cond_cmp_sel( +; CHECK-LABEL: define float @cond_cmp_sel( +; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[COND:%.*]], i64 [[N:%.*]]) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[PRED_LOAD_CONTINUE6]] ] -; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[COND:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi float [ 1.000000e+00, [[VECTOR_PH]] ], [ [[RDX_MINMAX_SELECT:%.*]], [[PRED_LOAD_CONTINUE6]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = fcmp une <4 x float> [[WIDE_LOAD]], -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0 -; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = fcmp une <4 x float> [[WIDE_LOAD]], +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 +; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] ; CHECK: pred.load.if: -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[TMP5:%.*]] = load float, ptr [[TMP4]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x float> poison, float [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP4:%.*]] = load float, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x float> poison, float [[TMP4]], i64 0 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] ; CHECK: pred.load.continue: -; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ] -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1 -; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] +; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 +; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] ; CHECK: pred.load.if1: -; CHECK-NEXT: [[TMP9:%.*]] = or disjoint i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP9]] -; CHECK-NEXT: [[TMP11:%.*]] = load float, ptr [[TMP10]], align 4 -; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x float> [[TMP7]], float [[TMP11]], i64 1 +; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = load float, ptr [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x float> [[TMP6]], float [[TMP10]], i64 1 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]] ; CHECK: pred.load.continue2: -; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x float> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ] -; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2 -; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] +; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x float> [ [[TMP6]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], [[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 +; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] ; CHECK: pred.load.if3: -; CHECK-NEXT: [[TMP15:%.*]] = or disjoint i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[TMP16]], align 4 -; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> [[TMP13]], float [[TMP17]], i64 2 +; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = load float, ptr [[TMP15]], align 4 +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x float> [[TMP12]], float [[TMP16]], i64 2 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]] ; CHECK: pred.load.continue4: -; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x float> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ] -; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3 -; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] +; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x float> [ [[TMP12]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP17]], [[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 +; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.if5: -; CHECK-NEXT: [[TMP21:%.*]] = or disjoint i64 [[INDEX]], 3 -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = load float, ptr [[TMP22]], align 4 -; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x float> [[TMP19]], float [[TMP23]], i64 3 +; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[TMP20]] +; CHECK-NEXT: [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x float> [[TMP18]], float [[TMP22]], i64 3 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.continue6: -; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x float> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ] -; CHECK-NEXT: [[TMP26:%.*]] = select fast <4 x i1> [[TMP2]], <4 x float> [[TMP25]], <4 x float> -; CHECK-NEXT: [[TMP27:%.*]] = call fast float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP26]]) -; CHECK-NEXT: [[TMP28]] = call fast float @llvm.minnum.f32(float [[TMP27]], float [[VEC_PHI]]) +; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x float> [ [[TMP18]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP25:%.*]] = select fast <4 x i1> [[TMP1]], <4 x float> [[TMP24]], <4 x float> +; CHECK-NEXT: [[TMP26:%.*]] = call fast float @llvm.vector.reduce.fmin.v4f32(<4 x float> [[TMP25]]) +; CHECK-NEXT: [[RDX_MINMAX_SELECT]] = call fast float @llvm.minnum.f32(float [[TMP26]], float [[VEC_PHI]]) ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] +; CHECK-NEXT: [[TMP27:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP27]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP28]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ], [ 1.000000e+00, [[ENTRY]] ] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ] ; CHECK-NEXT: [[RDX:%.*]] = phi float [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[IV]] -; CHECK-NEXT: [[TMP30:%.*]] = load float, ptr [[ARRAYIDX]], align 4 -; CHECK-NEXT: [[TOBOOL:%.*]] = fcmp une float [[TMP30]], 3.000000e+00 +; CHECK-NEXT: [[TMP28:%.*]] = load float, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[TOBOOL:%.*]] = fcmp une float [[TMP28]], 3.000000e+00 ; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]] ; CHECK: if.then: ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[A]], i64 [[IV]] -; CHECK-NEXT: [[TMP31:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 -; CHECK-NEXT: [[TMP32:%.*]] = call fast float @llvm.minnum.f32(float [[RDX]], float [[TMP31]]) +; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[FSEL:%.*]] = call fast float @llvm.minnum.f32(float [[RDX]], float [[TMP29]]) ; CHECK-NEXT: br label [[FOR_INC]] ; CHECK: for.inc: -; CHECK-NEXT: [[RES]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[TMP32]], [[IF_THEN]] ] +; CHECK-NEXT: [[RES]] = phi float [ [[RDX]], [[FOR_BODY]] ], [ [[FSEL]], [[IF_THEN]] ] ; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] ; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] ; CHECK: for.end: -; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP28]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[RDX_MINMAX_SELECT]], [[MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret float [[RES_LCSSA]] ; entry: @@ -229,84 +232,85 @@ for.end: } define i32 @conditional_and(ptr noalias %A, ptr noalias %B, i32 %cond, i64 noundef %N) #0 { -; CHECK-LABEL: @conditional_and( +; CHECK-LABEL: define i32 @conditional_and( +; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], i32 [[COND:%.*]], i64 noundef [[N:%.*]]) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[COND:%.*]], i64 0 +; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i32> poison, i32 [[COND]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ] -; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 7, [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[PRED_LOAD_CONTINUE6]] ] -; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 7, [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[PRED_LOAD_CONTINUE6]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 -; CHECK-NEXT: [[TMP2:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] -; CHECK-NEXT: [[TMP3:%.*]] = extractelement <4 x i1> [[TMP2]], i64 0 -; CHECK-NEXT: br i1 [[TMP3]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK-NEXT: [[TMP1:%.*]] = icmp eq <4 x i32> [[WIDE_LOAD]], [[BROADCAST_SPLAT]] +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 +; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] ; CHECK: pred.load.if: -; CHECK-NEXT: [[TMP4:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[TMP5:%.*]] = load i32, ptr [[TMP4]], align 4 -; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] ; CHECK: pred.load.continue: -; CHECK-NEXT: [[TMP7:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP6]], [[PRED_LOAD_IF]] ] -; CHECK-NEXT: [[TMP8:%.*]] = extractelement <4 x i1> [[TMP2]], i64 1 -; CHECK-NEXT: br i1 [[TMP8]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] +; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 +; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] ; CHECK: pred.load.if1: -; CHECK-NEXT: [[TMP9:%.*]] = or disjoint i64 [[INDEX]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP9]] -; CHECK-NEXT: [[TMP11:%.*]] = load i32, ptr [[TMP10]], align 4 -; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[TMP11]], i64 1 +; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP10]], i64 1 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]] ; CHECK: pred.load.continue2: -; CHECK-NEXT: [[TMP13:%.*]] = phi <4 x i32> [ [[TMP7]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP12]], [[PRED_LOAD_IF1]] ] -; CHECK-NEXT: [[TMP14:%.*]] = extractelement <4 x i1> [[TMP2]], i64 2 -; CHECK-NEXT: br i1 [[TMP14]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] +; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ [[TMP6]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], [[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 +; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] ; CHECK: pred.load.if3: -; CHECK-NEXT: [[TMP15:%.*]] = or disjoint i64 [[INDEX]], 2 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP15]] -; CHECK-NEXT: [[TMP17:%.*]] = load i32, ptr [[TMP16]], align 4 -; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i32> [[TMP13]], i32 [[TMP17]], i64 2 +; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4 +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP16]], i64 2 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]] ; CHECK: pred.load.continue4: -; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i32> [ [[TMP13]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ] -; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP2]], i64 3 -; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] +; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP12]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP17]], [[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 +; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.if5: -; CHECK-NEXT: [[TMP21:%.*]] = or disjoint i64 [[INDEX]], 3 -; CHECK-NEXT: [[TMP22:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP21]] -; CHECK-NEXT: [[TMP23:%.*]] = load i32, ptr [[TMP22]], align 4 -; CHECK-NEXT: [[TMP24:%.*]] = insertelement <4 x i32> [[TMP19]], i32 [[TMP23]], i64 3 +; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[TMP20]] +; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP22]], i64 3 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.continue6: -; CHECK-NEXT: [[TMP25:%.*]] = phi <4 x i32> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP24]], [[PRED_LOAD_IF5]] ] -; CHECK-NEXT: [[TMP26:%.*]] = select <4 x i1> [[TMP2]], <4 x i32> [[TMP25]], <4 x i32> -; CHECK-NEXT: [[TMP27:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP26]]) -; CHECK-NEXT: [[TMP28]] = and i32 [[TMP27]], [[VEC_PHI]] +; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i32> [ [[TMP18]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP24]], <4 x i32> +; CHECK-NEXT: [[TMP26:%.*]] = call i32 @llvm.vector.reduce.and.v4i32(<4 x i32> [[TMP25]]) +; CHECK-NEXT: [[TMP27]] = and i32 [[TMP26]], [[VEC_PHI]] ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 -; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] -; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] +; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP6:![0-9]+]] ; CHECK: middle.block: ; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] ; CHECK: scalar.ph: ; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] -; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP28]], [[MIDDLE_BLOCK]] ], [ 7, [[ENTRY]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP27]], [[MIDDLE_BLOCK]] ], [ 7, [[ENTRY]] ] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ] ; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[IV]] -; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 -; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[TMP30]], [[COND]] +; CHECK-NEXT: [[TMP29:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i32 [[TMP29]], [[COND]] ; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]] ; CHECK: if.then: ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[IV]] -; CHECK-NEXT: [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 -; CHECK-NEXT: [[AND:%.*]] = and i32 [[TMP31]], [[RDX]] +; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[AND:%.*]] = and i32 [[TMP30]], [[RDX]] ; CHECK-NEXT: br label [[FOR_INC]] ; CHECK: for.inc: ; CHECK-NEXT: [[RES]] = phi i32 [ [[AND]], [[IF_THEN]] ], [ [[RDX]], [[FOR_BODY]] ] @@ -314,7 +318,7 @@ define i32 @conditional_and(ptr noalias %A, ptr noalias %B, i32 %cond, i64 nound ; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] ; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] ; CHECK: for.end: -; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_INC]] ], [ [[TMP28]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_INC]] ], [ [[TMP27]], [[MIDDLE_BLOCK]] ] ; CHECK-NEXT: ret i32 [[RES_LCSSA]] ; entry: @@ -345,9 +349,10 @@ for.end: } define i32 @simple_chained_rdx(ptr noalias %a, ptr noalias %b, ptr noalias %cond, i64 noundef %N) { -; CHECK-LABEL: @simple_chained_rdx( +; CHECK-LABEL: define i32 @simple_chained_rdx( +; CHECK-SAME: ptr noalias [[A:%.*]], ptr noalias [[B:%.*]], ptr noalias [[COND:%.*]], i64 noundef [[N:%.*]]) { ; CHECK-NEXT: entry: -; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N:%.*]], 4 +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 ; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] ; CHECK: vector.ph: ; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 @@ -355,16 +360,16 @@ define i32 @simple_chained_rdx(ptr noalias %a, ptr noalias %b, ptr noalias %cond ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi i32 [ 5, [[VECTOR_PH]] ], [ [[TMP46:%.*]], [[PRED_LOAD_CONTINUE6]] ] -; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]] ; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 ; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer ; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 ; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] ; CHECK: pred.load.if: -; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[A]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 ; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[B]], i64 [[INDEX]] ; CHECK-NEXT: [[TMP7:%.*]] = load i32, ptr [[TMP6]], align 4 ; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> poison, i32 [[TMP7]], i64 0 ; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] @@ -491,12 +496,140 @@ for.end: ; Reduction not performed in loop as the phi has more than two incoming values ; define i64 @nested_cond_and(ptr noalias nocapture readonly %a, ptr noalias nocapture readonly %b, ptr noalias nocapture readonly %cond, i64 %N){ -; CHECK-LABEL: @nested_cond_and( +; CHECK-LABEL: define i64 @nested_cond_and( +; CHECK-SAME: ptr noalias nocapture readonly [[A:%.*]], ptr noalias nocapture readonly [[B:%.*]], ptr noalias nocapture readonly [[COND:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: -; CHECK-NOT: @llvm.vector.reduce.and +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE14:%.*]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i64> [ , [[VECTOR_PH]] ], [ [[PREDPHI15:%.*]], [[PRED_LOAD_CONTINUE14]] ] +; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i64, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i64>, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = icmp eq <4 x i64> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0 +; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK: pred.load.if: +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP7:%.*]] = load i64, ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i64> poison, i64 [[TMP7]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] +; CHECK: pred.load.continue: +; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x i64> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1 +; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] +; CHECK: pred.load.if1: +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP0]] +; CHECK-NEXT: [[TMP12:%.*]] = load i64, ptr [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i64> [[TMP9]], i64 [[TMP12]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]] +; CHECK: pred.load.continue2: +; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i64> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2 +; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] +; CHECK: pred.load.if3: +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP1]] +; CHECK-NEXT: [[TMP17:%.*]] = load i64, ptr [[TMP16]], align 4 +; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x i64> [[TMP14]], i64 [[TMP17]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]] +; CHECK: pred.load.continue4: +; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x i64> [ [[TMP14]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3 +; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]] +; CHECK: pred.load.if5: +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[TMP2]] +; CHECK-NEXT: [[TMP22:%.*]] = load i64, ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i64> [[TMP19]], i64 [[TMP22]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]] +; CHECK: pred.load.continue6: +; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i64> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP25:%.*]] = icmp eq <4 x i64> [[TMP24]], +; CHECK-NEXT: [[TMP26:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP25]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP26]], i64 0 +; CHECK-NEXT: br i1 [[TMP27]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]] +; CHECK: pred.load.if7: +; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP29:%.*]] = load i64, ptr [[TMP28]], align 4 +; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x i64> poison, i64 [[TMP29]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE8]] +; CHECK: pred.load.continue8: +; CHECK-NEXT: [[TMP31:%.*]] = phi <4 x i64> [ poison, [[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], [[PRED_LOAD_IF7]] ] +; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP26]], i64 1 +; CHECK-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]] +; CHECK: pred.load.if9: +; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP0]] +; CHECK-NEXT: [[TMP34:%.*]] = load i64, ptr [[TMP33]], align 4 +; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x i64> [[TMP31]], i64 [[TMP34]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE10]] +; CHECK: pred.load.continue10: +; CHECK-NEXT: [[TMP36:%.*]] = phi <4 x i64> [ [[TMP31]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP35]], [[PRED_LOAD_IF9]] ] +; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP26]], i64 2 +; CHECK-NEXT: br i1 [[TMP37]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]] +; CHECK: pred.load.if11: +; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP1]] +; CHECK-NEXT: [[TMP39:%.*]] = load i64, ptr [[TMP38]], align 4 +; CHECK-NEXT: [[TMP40:%.*]] = insertelement <4 x i64> [[TMP36]], i64 [[TMP39]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE12]] +; CHECK: pred.load.continue12: +; CHECK-NEXT: [[TMP41:%.*]] = phi <4 x i64> [ [[TMP36]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP40]], [[PRED_LOAD_IF11]] ] +; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i1> [[TMP26]], i64 3 +; CHECK-NEXT: br i1 [[TMP42]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14]] +; CHECK: pred.load.if13: +; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[TMP2]] +; CHECK-NEXT: [[TMP44:%.*]] = load i64, ptr [[TMP43]], align 4 +; CHECK-NEXT: [[TMP45:%.*]] = insertelement <4 x i64> [[TMP41]], i64 [[TMP44]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE14]] +; CHECK: pred.load.continue14: +; CHECK-NEXT: [[TMP46:%.*]] = phi <4 x i64> [ [[TMP41]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP45]], [[PRED_LOAD_IF13]] ] +; CHECK-NEXT: [[TMP47:%.*]] = xor <4 x i1> [[TMP25]], +; CHECK-NEXT: [[TMP48:%.*]] = select <4 x i1> [[TMP4]], <4 x i1> [[TMP47]], <4 x i1> zeroinitializer +; CHECK-NEXT: [[PREDPHI_V:%.*]] = select <4 x i1> [[TMP48]], <4 x i64> [[TMP24]], <4 x i64> [[TMP46]] +; CHECK-NEXT: [[PREDPHI:%.*]] = select <4 x i1> [[TMP4]], <4 x i64> [[PREDPHI_V]], <4 x i64> +; CHECK-NEXT: [[PREDPHI15]] = and <4 x i64> [[VEC_PHI]], [[PREDPHI]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP49:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP49]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: middle.block: -; CHECK: @llvm.vector.reduce.and -; CHECK: scalar.ph +; CHECK-NEXT: [[TMP50:%.*]] = call i64 @llvm.vector.reduce.and.v4i64(<4 x i64> [[PREDPHI15]]) +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i64 [ [[TMP50]], [[MIDDLE_BLOCK]] ], [ 5, [[ENTRY]] ] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[IV_NEXT:%.*]], [[FOR_INC:%.*]] ] +; CHECK-NEXT: [[RDX:%.*]] = phi i64 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[RES:%.*]], [[FOR_INC]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i64, ptr [[COND]], i64 [[IV]] +; CHECK-NEXT: [[TMP51:%.*]] = load i64, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[TOBOOL:%.*]] = icmp eq i64 [[TMP51]], 0 +; CHECK-NEXT: br i1 [[TOBOOL]], label [[IF_THEN:%.*]], label [[FOR_INC]] +; CHECK: if.then: +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i64, ptr [[A]], i64 [[IV]] +; CHECK-NEXT: [[TMP52:%.*]] = load i64, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[AND1:%.*]] = and i64 [[RDX]], [[TMP52]] +; CHECK-NEXT: [[TOBOOL2:%.*]] = icmp eq i64 [[TMP52]], 3 +; CHECK-NEXT: br i1 [[TOBOOL2]], label [[IF_THEN_2:%.*]], label [[FOR_INC]] +; CHECK: if.then.2: +; CHECK-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds i64, ptr [[B]], i64 [[IV]] +; CHECK-NEXT: [[TMP53:%.*]] = load i64, ptr [[ARRAYIDX3]], align 4 +; CHECK-NEXT: [[AND2:%.*]] = and i64 [[RDX]], [[TMP53]] +; CHECK-NEXT: br label [[FOR_INC]] +; CHECK: for.inc: +; CHECK-NEXT: [[RES]] = phi i64 [ [[AND2]], [[IF_THEN_2]] ], [ [[AND1]], [[IF_THEN]] ], [ [[RDX]], [[FOR_BODY]] ] +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP11:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i64 [ [[RES]], [[FOR_INC]] ], [ [[TMP50]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret i64 [[RES_LCSSA]] +; entry: br label %for.body @@ -536,12 +669,99 @@ for.end: ; the Phi (%rdx1) as it has more than one use. ; define i32 @cond-uncond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond, i64 noundef %n) #0 { -; CHECK-LABEL: @cond-uncond( +; CHECK-LABEL: define i32 @cond-uncond( +; CHECK-SAME: ptr noalias [[SRC1:%.*]], ptr noalias [[SRC2:%.*]], ptr noalias [[COND:%.*]], i64 noundef [[N:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE6:%.*]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP27:%.*]], [[PRED_LOAD_CONTINUE6]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD]], zeroinitializer +; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i1> [[TMP1]], i64 0 +; CHECK-NEXT: br i1 [[TMP2]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK: pred.load.if: +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP4:%.*]] = load i32, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <4 x i32> poison, i32 [[TMP4]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] +; CHECK: pred.load.continue: +; CHECK-NEXT: [[TMP6:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP5]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <4 x i1> [[TMP1]], i64 1 +; CHECK-NEXT: br i1 [[TMP7]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] +; CHECK: pred.load.if1: +; CHECK-NEXT: [[TMP8:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP8]] +; CHECK-NEXT: [[TMP10:%.*]] = load i32, ptr [[TMP9]], align 4 +; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[TMP10]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]] +; CHECK: pred.load.continue2: +; CHECK-NEXT: [[TMP12:%.*]] = phi <4 x i32> [ [[TMP6]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP11]], [[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP13:%.*]] = extractelement <4 x i1> [[TMP1]], i64 2 +; CHECK-NEXT: br i1 [[TMP13]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] +; CHECK: pred.load.if3: +; CHECK-NEXT: [[TMP14:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP14]] +; CHECK-NEXT: [[TMP16:%.*]] = load i32, ptr [[TMP15]], align 4 +; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x i32> [[TMP12]], i32 [[TMP16]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]] +; CHECK: pred.load.continue4: +; CHECK-NEXT: [[TMP18:%.*]] = phi <4 x i32> [ [[TMP12]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP17]], [[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP19:%.*]] = extractelement <4 x i1> [[TMP1]], i64 3 +; CHECK-NEXT: br i1 [[TMP19]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6]] +; CHECK: pred.load.if5: +; CHECK-NEXT: [[TMP20:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP20]] +; CHECK-NEXT: [[TMP22:%.*]] = load i32, ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x i32> [[TMP18]], i32 [[TMP22]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]] ; CHECK: pred.load.continue6: -; CHECK-NOT: @llvm.vector.reduce.add +; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x i32> [ [[TMP18]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP1]], <4 x i32> [[TMP24]], <4 x i32> zeroinitializer +; CHECK-NEXT: [[PREDPHI:%.*]] = add <4 x i32> [[VEC_PHI]], [[TMP25]] +; CHECK-NEXT: [[TMP26:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD7:%.*]] = load <4 x i32>, ptr [[TMP26]], align 4 +; CHECK-NEXT: [[TMP27]] = add <4 x i32> [[WIDE_LOAD7]], [[PREDPHI]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] ; CHECK: middle.block: -; CHECK: @llvm.vector.reduce.add -; CHECK: scalar.ph +; CHECK-NEXT: [[TMP29:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP27]]) +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP29]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[RDX1:%.*]] = phi i32 [ [[ADD2:%.*]], [[IF_END:%.*]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[IF_END]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[IV]] +; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP30]], 0 +; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[IV]] +; CHECK-NEXT: [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4 +; CHECK-NEXT: [[ADD:%.*]] = add nsw i32 [[TMP31]], [[RDX1]] +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[ADD]], [[IF_THEN]] ], [ [[RDX1]], [[FOR_BODY]] ] +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[IV]] +; CHECK-NEXT: [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[ADD2]] = add nsw i32 [[TMP32]], [[RES]] +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP13:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: [[ADD2_LCSSA:%.*]] = phi i32 [ [[ADD2]], [[IF_END]] ], [ [[TMP29]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret i32 [[ADD2_LCSSA]] +; entry: br label %for.body @@ -577,12 +797,141 @@ for.end: ; of the incoming values of the LoopExitInstruction (%res) is the reduction Phi (%rdx1). ; define float @cond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond, i64 %n) #0 { -; CHECK-LABEL: @cond_cond( +; CHECK-LABEL: define float @cond_cond( +; CHECK-SAME: ptr noalias [[SRC1:%.*]], ptr noalias [[SRC2:%.*]], ptr noalias [[COND:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE14:%.*]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x float> [ , [[VECTOR_PH]] ], [ [[PREDPHI15:%.*]], [[PRED_LOAD_CONTINUE14]] ] +; CHECK-NEXT: [[TMP0:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP1:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP2:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP3:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x float>, ptr [[TMP3]], align 4 +; CHECK-NEXT: [[TMP4:%.*]] = fcmp fast oeq <4 x float> [[WIDE_LOAD]], +; CHECK-NEXT: [[TMP5:%.*]] = extractelement <4 x i1> [[TMP4]], i64 0 +; CHECK-NEXT: br i1 [[TMP5]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK: pred.load.if: +; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP7:%.*]] = load float, ptr [[TMP6]], align 4 +; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x float> poison, float [[TMP7]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] +; CHECK: pred.load.continue: +; CHECK-NEXT: [[TMP9:%.*]] = phi <4 x float> [ poison, [[VECTOR_BODY]] ], [ [[TMP8]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP10:%.*]] = extractelement <4 x i1> [[TMP4]], i64 1 +; CHECK-NEXT: br i1 [[TMP10]], label [[PRED_LOAD_IF1:%.*]], label [[PRED_LOAD_CONTINUE2:%.*]] +; CHECK: pred.load.if1: +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[TMP0]] +; CHECK-NEXT: [[TMP12:%.*]] = load float, ptr [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x float> [[TMP9]], float [[TMP12]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE2]] +; CHECK: pred.load.continue2: +; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x float> [ [[TMP9]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF1]] ] +; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP4]], i64 2 +; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF3:%.*]], label [[PRED_LOAD_CONTINUE4:%.*]] +; CHECK: pred.load.if3: +; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[TMP1]] +; CHECK-NEXT: [[TMP17:%.*]] = load float, ptr [[TMP16]], align 4 +; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> [[TMP14]], float [[TMP17]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE4]] +; CHECK: pred.load.continue4: +; CHECK-NEXT: [[TMP19:%.*]] = phi <4 x float> [ [[TMP14]], [[PRED_LOAD_CONTINUE2]] ], [ [[TMP18]], [[PRED_LOAD_IF3]] ] +; CHECK-NEXT: [[TMP20:%.*]] = extractelement <4 x i1> [[TMP4]], i64 3 +; CHECK-NEXT: br i1 [[TMP20]], label [[PRED_LOAD_IF5:%.*]], label [[PRED_LOAD_CONTINUE6:%.*]] +; CHECK: pred.load.if5: +; CHECK-NEXT: [[TMP21:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[TMP2]] +; CHECK-NEXT: [[TMP22:%.*]] = load float, ptr [[TMP21]], align 4 +; CHECK-NEXT: [[TMP23:%.*]] = insertelement <4 x float> [[TMP19]], float [[TMP22]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE6]] +; CHECK: pred.load.continue6: +; CHECK-NEXT: [[TMP24:%.*]] = phi <4 x float> [ [[TMP19]], [[PRED_LOAD_CONTINUE4]] ], [ [[TMP23]], [[PRED_LOAD_IF5]] ] +; CHECK-NEXT: [[TMP25:%.*]] = select <4 x i1> [[TMP4]], <4 x float> [[TMP24]], <4 x float> +; CHECK-NEXT: [[PREDPHI:%.*]] = fadd fast <4 x float> [[VEC_PHI]], [[TMP25]] +; CHECK-NEXT: [[TMP26:%.*]] = fcmp fast oeq <4 x float> [[WIDE_LOAD]], +; CHECK-NEXT: [[TMP27:%.*]] = extractelement <4 x i1> [[TMP26]], i64 0 +; CHECK-NEXT: br i1 [[TMP27]], label [[PRED_LOAD_IF7:%.*]], label [[PRED_LOAD_CONTINUE8:%.*]] +; CHECK: pred.load.if7: +; CHECK-NEXT: [[TMP28:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP29:%.*]] = load float, ptr [[TMP28]], align 4 +; CHECK-NEXT: [[TMP30:%.*]] = insertelement <4 x float> poison, float [[TMP29]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE8]] +; CHECK: pred.load.continue8: +; CHECK-NEXT: [[TMP31:%.*]] = phi <4 x float> [ poison, [[PRED_LOAD_CONTINUE6]] ], [ [[TMP30]], [[PRED_LOAD_IF7]] ] +; CHECK-NEXT: [[TMP32:%.*]] = extractelement <4 x i1> [[TMP26]], i64 1 +; CHECK-NEXT: br i1 [[TMP32]], label [[PRED_LOAD_IF9:%.*]], label [[PRED_LOAD_CONTINUE10:%.*]] +; CHECK: pred.load.if9: +; CHECK-NEXT: [[TMP33:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[TMP0]] +; CHECK-NEXT: [[TMP34:%.*]] = load float, ptr [[TMP33]], align 4 +; CHECK-NEXT: [[TMP35:%.*]] = insertelement <4 x float> [[TMP31]], float [[TMP34]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE10]] +; CHECK: pred.load.continue10: +; CHECK-NEXT: [[TMP36:%.*]] = phi <4 x float> [ [[TMP31]], [[PRED_LOAD_CONTINUE8]] ], [ [[TMP35]], [[PRED_LOAD_IF9]] ] +; CHECK-NEXT: [[TMP37:%.*]] = extractelement <4 x i1> [[TMP26]], i64 2 +; CHECK-NEXT: br i1 [[TMP37]], label [[PRED_LOAD_IF11:%.*]], label [[PRED_LOAD_CONTINUE12:%.*]] +; CHECK: pred.load.if11: +; CHECK-NEXT: [[TMP38:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[TMP1]] +; CHECK-NEXT: [[TMP39:%.*]] = load float, ptr [[TMP38]], align 4 +; CHECK-NEXT: [[TMP40:%.*]] = insertelement <4 x float> [[TMP36]], float [[TMP39]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE12]] +; CHECK: pred.load.continue12: +; CHECK-NEXT: [[TMP41:%.*]] = phi <4 x float> [ [[TMP36]], [[PRED_LOAD_CONTINUE10]] ], [ [[TMP40]], [[PRED_LOAD_IF11]] ] +; CHECK-NEXT: [[TMP42:%.*]] = extractelement <4 x i1> [[TMP26]], i64 3 +; CHECK-NEXT: br i1 [[TMP42]], label [[PRED_LOAD_IF13:%.*]], label [[PRED_LOAD_CONTINUE14]] +; CHECK: pred.load.if13: +; CHECK-NEXT: [[TMP43:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[TMP2]] +; CHECK-NEXT: [[TMP44:%.*]] = load float, ptr [[TMP43]], align 4 +; CHECK-NEXT: [[TMP45:%.*]] = insertelement <4 x float> [[TMP41]], float [[TMP44]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE14]] ; CHECK: pred.load.continue14: -; CHECK-NOT: @llvm.vector.reduce.fadd +; CHECK-NEXT: [[TMP46:%.*]] = phi <4 x float> [ [[TMP41]], [[PRED_LOAD_CONTINUE12]] ], [ [[TMP45]], [[PRED_LOAD_IF13]] ] +; CHECK-NEXT: [[TMP47:%.*]] = select <4 x i1> [[TMP26]], <4 x float> [[TMP46]], <4 x float> +; CHECK-NEXT: [[PREDPHI15]] = fadd fast <4 x float> [[PREDPHI]], [[TMP47]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP48:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP48]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP14:![0-9]+]] ; CHECK: middle.block: -; CHECK: @llvm.vector.reduce.fadd -; CHECK: scalar.ph +; CHECK-NEXT: [[TMP49:%.*]] = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.000000e+00, <4 x float> [[PREDPHI15]]) +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi float [ [[TMP49]], [[MIDDLE_BLOCK]] ], [ 2.000000e+00, [[ENTRY]] ] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[RDX1:%.*]] = phi float [ [[RES:%.*]], [[FOR_INC:%.*]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds float, ptr [[COND]], i64 [[IV]] +; CHECK-NEXT: [[TMP50:%.*]] = load float, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[CMP1:%.*]] = fcmp fast oeq float [[TMP50]], 3.000000e+00 +; CHECK-NEXT: br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds float, ptr [[SRC1]], i64 [[IV]] +; CHECK-NEXT: [[TMP51:%.*]] = load float, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP51]], [[RDX1]] +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[RDX2:%.*]] = phi float [ [[ADD]], [[IF_THEN]] ], [ [[RDX1]], [[FOR_BODY]] ] +; CHECK-NEXT: [[CMP5:%.*]] = fcmp fast oeq float [[TMP50]], 7.000000e+00 +; CHECK-NEXT: br i1 [[CMP5]], label [[IF_THEN6:%.*]], label [[FOR_INC]] +; CHECK: if.then6: +; CHECK-NEXT: [[ARRAYIDX7:%.*]] = getelementptr inbounds float, ptr [[SRC2]], i64 [[IV]] +; CHECK-NEXT: [[TMP52:%.*]] = load float, ptr [[ARRAYIDX7]], align 4 +; CHECK-NEXT: [[ADD2:%.*]] = fadd fast float [[TMP52]], [[RDX2]] +; CHECK-NEXT: br label [[FOR_INC]] +; CHECK: for.inc: +; CHECK-NEXT: [[RES]] = phi float [ [[ADD2]], [[IF_THEN6]] ], [ [[RDX2]], [[IF_END]] ] +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP15:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi float [ [[RES]], [[FOR_INC]] ], [ [[TMP49]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret float [[RES_LCSSA]] +; entry: br label %for.body @@ -626,12 +975,99 @@ for.end: ; incoming values of the LoopExitInstruction (%res) is the reduction Phi (%rdx). ; define i32 @uncond_cond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond, i64 %N) #0 { -; CHECK-LABEL: @uncond_cond( +; CHECK-LABEL: define i32 @uncond_cond( +; CHECK-SAME: ptr noalias [[SRC1:%.*]], ptr noalias [[SRC2:%.*]], ptr noalias [[COND:%.*]], i64 [[N:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE7:%.*]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[PREDPHI:%.*]], [[PRED_LOAD_CONTINUE7]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]] +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD1]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0 +; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK: pred.load.if: +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] +; CHECK: pred.load.continue: +; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1 +; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]] +; CHECK: pred.load.if2: +; CHECK-NEXT: [[TMP10:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP12]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE3]] +; CHECK: pred.load.continue3: +; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP8]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF2]] ] +; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2 +; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]] +; CHECK: pred.load.if4: +; CHECK-NEXT: [[TMP16:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP16]] +; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4 +; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP18]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE5]] +; CHECK: pred.load.continue5: +; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_LOAD_CONTINUE3]] ], [ [[TMP19]], [[PRED_LOAD_IF4]] ] +; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3 +; CHECK-NEXT: br i1 [[TMP21]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7]] +; CHECK: pred.load.if6: +; CHECK-NEXT: [[TMP22:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP22]] +; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4 +; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP24]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE7]] ; CHECK: pred.load.continue7: -; CHECK-NOT: @llvm.vector.reduce.add +; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ [[TMP20]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP25]], [[PRED_LOAD_IF6]] ] +; CHECK-NEXT: [[TMP27:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP26]], <4 x i32> zeroinitializer +; CHECK-NEXT: [[PREDPHI]] = add <4 x i32> [[TMP1]], [[TMP27]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP28:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP28]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP16:![0-9]+]] ; CHECK: middle.block: -; CHECK: @llvm.vector.reduce.add -; CHECK: scalar.ph +; CHECK-NEXT: [[TMP29:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[PREDPHI]]) +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP29]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ [[RES:%.*]], [[FOR_INC:%.*]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[FOR_INC]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[IV]] +; CHECK-NEXT: [[TMP30:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[ADD1:%.*]] = add nsw i32 [[TMP30]], [[RDX]] +; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[IV]] +; CHECK-NEXT: [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4 +; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP31]], 0 +; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label [[FOR_INC]], label [[IF_THEN:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[IV]] +; CHECK-NEXT: [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP32]], [[ADD1]] +; CHECK-NEXT: br label [[FOR_INC]] +; CHECK: for.inc: +; CHECK-NEXT: [[RES]] = phi i32 [ [[ADD2]], [[IF_THEN]] ], [ [[ADD1]], [[FOR_BODY]] ] +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP17:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: [[RES_LCSSA:%.*]] = phi i32 [ [[RES]], [[FOR_INC]] ], [ [[TMP29]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret i32 [[RES_LCSSA]] +; entry: br label %for.body @@ -667,12 +1103,101 @@ for.end: ; through the chain and check the number of uses of %add1, we find more than the expected one use. ; define i32 @uncond_cond_uncond(ptr noalias %src1, ptr noalias %src2, ptr noalias %cond, i64 noundef %N) { -; CHECK-LABEL: @uncond_cond_uncond( +; CHECK-LABEL: define i32 @uncond_cond_uncond( +; CHECK-SAME: ptr noalias [[SRC1:%.*]], ptr noalias [[SRC2:%.*]], ptr noalias [[COND:%.*]], i64 noundef [[N:%.*]]) { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[MIN_ITERS_CHECK:%.*]] = icmp ult i64 [[N]], 4 +; CHECK-NEXT: br i1 [[MIN_ITERS_CHECK]], label [[SCALAR_PH:%.*]], label [[VECTOR_PH:%.*]] +; CHECK: vector.ph: +; CHECK-NEXT: [[N_VEC:%.*]] = and i64 [[N]], -4 +; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] +; CHECK: vector.body: +; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[PRED_LOAD_CONTINUE7:%.*]] ] +; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP28:%.*]], [[PRED_LOAD_CONTINUE7]] ] +; CHECK-NEXT: [[TMP0:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load <4 x i32>, ptr [[TMP0]], align 4 +; CHECK-NEXT: [[TMP1:%.*]] = add <4 x i32> [[WIDE_LOAD]], [[VEC_PHI]] +; CHECK-NEXT: [[TMP2:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD1:%.*]] = load <4 x i32>, ptr [[TMP2]], align 4 +; CHECK-NEXT: [[TMP3:%.*]] = icmp ne <4 x i32> [[WIDE_LOAD1]], zeroinitializer +; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x i1> [[TMP3]], i64 0 +; CHECK-NEXT: br i1 [[TMP4]], label [[PRED_LOAD_IF:%.*]], label [[PRED_LOAD_CONTINUE:%.*]] +; CHECK: pred.load.if: +; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[INDEX]] +; CHECK-NEXT: [[TMP6:%.*]] = load i32, ptr [[TMP5]], align 4 +; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[TMP6]], i64 0 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE]] +; CHECK: pred.load.continue: +; CHECK-NEXT: [[TMP8:%.*]] = phi <4 x i32> [ poison, [[VECTOR_BODY]] ], [ [[TMP7]], [[PRED_LOAD_IF]] ] +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <4 x i1> [[TMP3]], i64 1 +; CHECK-NEXT: br i1 [[TMP9]], label [[PRED_LOAD_IF2:%.*]], label [[PRED_LOAD_CONTINUE3:%.*]] +; CHECK: pred.load.if2: +; CHECK-NEXT: [[TMP10:%.*]] = or disjoint i64 [[INDEX]], 1 +; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP10]] +; CHECK-NEXT: [[TMP12:%.*]] = load i32, ptr [[TMP11]], align 4 +; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[TMP12]], i64 1 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE3]] +; CHECK: pred.load.continue3: +; CHECK-NEXT: [[TMP14:%.*]] = phi <4 x i32> [ [[TMP8]], [[PRED_LOAD_CONTINUE]] ], [ [[TMP13]], [[PRED_LOAD_IF2]] ] +; CHECK-NEXT: [[TMP15:%.*]] = extractelement <4 x i1> [[TMP3]], i64 2 +; CHECK-NEXT: br i1 [[TMP15]], label [[PRED_LOAD_IF4:%.*]], label [[PRED_LOAD_CONTINUE5:%.*]] +; CHECK: pred.load.if4: +; CHECK-NEXT: [[TMP16:%.*]] = or disjoint i64 [[INDEX]], 2 +; CHECK-NEXT: [[TMP17:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP16]] +; CHECK-NEXT: [[TMP18:%.*]] = load i32, ptr [[TMP17]], align 4 +; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x i32> [[TMP14]], i32 [[TMP18]], i64 2 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE5]] +; CHECK: pred.load.continue5: +; CHECK-NEXT: [[TMP20:%.*]] = phi <4 x i32> [ [[TMP14]], [[PRED_LOAD_CONTINUE3]] ], [ [[TMP19]], [[PRED_LOAD_IF4]] ] +; CHECK-NEXT: [[TMP21:%.*]] = extractelement <4 x i1> [[TMP3]], i64 3 +; CHECK-NEXT: br i1 [[TMP21]], label [[PRED_LOAD_IF6:%.*]], label [[PRED_LOAD_CONTINUE7]] +; CHECK: pred.load.if6: +; CHECK-NEXT: [[TMP22:%.*]] = or disjoint i64 [[INDEX]], 3 +; CHECK-NEXT: [[TMP23:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[TMP22]] +; CHECK-NEXT: [[TMP24:%.*]] = load i32, ptr [[TMP23]], align 4 +; CHECK-NEXT: [[TMP25:%.*]] = insertelement <4 x i32> [[TMP20]], i32 [[TMP24]], i64 3 +; CHECK-NEXT: br label [[PRED_LOAD_CONTINUE7]] ; CHECK: pred.load.continue7: -; CHECK-NOT: @llvm.vector.reduce.add +; CHECK-NEXT: [[TMP26:%.*]] = phi <4 x i32> [ [[TMP20]], [[PRED_LOAD_CONTINUE5]] ], [ [[TMP25]], [[PRED_LOAD_IF6]] ] +; CHECK-NEXT: [[TMP27:%.*]] = select <4 x i1> [[TMP3]], <4 x i32> [[TMP26]], <4 x i32> zeroinitializer +; CHECK-NEXT: [[PREDPHI:%.*]] = add <4 x i32> [[TMP1]], [[TMP27]] +; CHECK-NEXT: [[TMP28]] = add <4 x i32> [[PREDPHI]], [[WIDE_LOAD]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], 4 +; CHECK-NEXT: [[TMP29:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] +; CHECK-NEXT: br i1 [[TMP29]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP18:![0-9]+]] ; CHECK: middle.block: -; CHECK: @llvm.vector.reduce.add -; CHECK: scalar.ph +; CHECK-NEXT: [[TMP30:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP28]]) +; CHECK-NEXT: [[CMP_N:%.*]] = icmp eq i64 [[N_VEC]], [[N]] +; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END:%.*]], label [[SCALAR_PH]] +; CHECK: scalar.ph: +; CHECK-NEXT: [[BC_RESUME_VAL:%.*]] = phi i64 [ [[N_VEC]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY:%.*]] ] +; CHECK-NEXT: [[BC_MERGE_RDX:%.*]] = phi i32 [ [[TMP30]], [[MIDDLE_BLOCK]] ], [ 0, [[ENTRY]] ] +; CHECK-NEXT: br label [[FOR_BODY:%.*]] +; CHECK: for.body: +; CHECK-NEXT: [[RDX:%.*]] = phi i32 [ [[ADD3:%.*]], [[IF_END:%.*]] ], [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[IV_NEXT:%.*]], [[IF_END]] ], [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ] +; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[SRC1]], i64 [[IV]] +; CHECK-NEXT: [[TMP31:%.*]] = load i32, ptr [[ARRAYIDX]], align 4 +; CHECK-NEXT: [[ADD1:%.*]] = add nsw i32 [[TMP31]], [[RDX]] +; CHECK-NEXT: [[ARRAYIDX1:%.*]] = getelementptr inbounds i32, ptr [[COND]], i64 [[IV]] +; CHECK-NEXT: [[TMP32:%.*]] = load i32, ptr [[ARRAYIDX1]], align 4 +; CHECK-NEXT: [[TOBOOL_NOT:%.*]] = icmp eq i32 [[TMP32]], 0 +; CHECK-NEXT: br i1 [[TOBOOL_NOT]], label [[IF_END]], label [[IF_THEN:%.*]] +; CHECK: if.then: +; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds i32, ptr [[SRC2]], i64 [[IV]] +; CHECK-NEXT: [[TMP33:%.*]] = load i32, ptr [[ARRAYIDX2]], align 4 +; CHECK-NEXT: [[ADD2:%.*]] = add nsw i32 [[TMP33]], [[ADD1]] +; CHECK-NEXT: br label [[IF_END]] +; CHECK: if.end: +; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[ADD2]], [[IF_THEN]] ], [ [[ADD1]], [[FOR_BODY]] ] +; CHECK-NEXT: [[ADD3]] = add nsw i32 [[RES]], [[TMP31]] +; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1 +; CHECK-NEXT: [[EXITCOND_NOT:%.*]] = icmp eq i64 [[IV_NEXT]], [[N]] +; CHECK-NEXT: br i1 [[EXITCOND_NOT]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP19:![0-9]+]] +; CHECK: for.end: +; CHECK-NEXT: [[ADD3_LCSSA:%.*]] = phi i32 [ [[ADD3]], [[IF_END]] ], [ [[TMP30]], [[MIDDLE_BLOCK]] ] +; CHECK-NEXT: ret i32 [[ADD3_LCSSA]] +; entry: br label %for.body diff --git a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll index 3973a288fe71..2a58748d8fb6 100644 --- a/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll +++ b/llvm/test/Transforms/LoopVectorize/reduction-small-size.ll @@ -11,15 +11,12 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) { ; CHECK: vector.ph: ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i32 [[N]], 4 ; CHECK-NEXT: [[N_VEC:%.*]] = sub i32 [[N]], [[N_MOD_VF]] -; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement <4 x i1> poison, i1 [[C:%.*]], i64 0 -; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector <4 x i1> [[BROADCAST_SPLATINSERT]], <4 x i1> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: [[BROADCAST_SPLATINSERT1:%.*]] = insertelement <4 x i32> poison, i32 [[X:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT2:%.*]] = shufflevector <4 x i32> [[BROADCAST_SPLATINSERT1]], <4 x i32> poison, <4 x i32> zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi <4 x i32> [ zeroinitializer, [[VECTOR_PH]] ], [ [[TMP4:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP0:%.*]] = select <4 x i1> [[BROADCAST_SPLAT]], <4 x i32> undef, <4 x i32> ; CHECK-NEXT: [[TMP1:%.*]] = and <4 x i32> [[VEC_PHI]], ; CHECK-NEXT: [[TMP2:%.*]] = add <4 x i32> [[TMP1]], [[BROADCAST_SPLAT2]] ; CHECK-NEXT: [[TMP3:%.*]] = trunc <4 x i32> [[TMP2]] to <4 x i8> @@ -40,7 +37,7 @@ define i8 @PR34687(i1 %c, i32 %x, i32 %n) { ; CHECK: for.body: ; CHECK-NEXT: [[I:%.*]] = phi i32 [ [[BC_RESUME_VAL]], [[SCALAR_PH]] ], [ [[I_NEXT:%.*]], [[IF_END:%.*]] ] ; CHECK-NEXT: [[R:%.*]] = phi i32 [ [[BC_MERGE_RDX]], [[SCALAR_PH]] ], [ [[R_NEXT:%.*]], [[IF_END]] ] -; CHECK-NEXT: br i1 [[C]], label [[IF_THEN:%.*]], label [[IF_END]] +; CHECK-NEXT: br i1 [[C:%.*]], label [[IF_THEN:%.*]], label [[IF_END]] ; CHECK: if.then: ; CHECK-NEXT: [[T0:%.*]] = sdiv i32 undef, undef ; CHECK-NEXT: br label [[IF_END]] diff --git a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll index 6d059c13597b..3be31c011eaa 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-first-order-recurrence.ll @@ -197,6 +197,7 @@ define i32 @extract_second_last_iteration(ptr %cval, i32 %x) { ; CHECK-VF4UF2: vector.ph ; CHECK-VF4UF2: call i32 @llvm.vscale.i32() ; CHECK-VF4UF2: call i32 @llvm.vscale.i32() +; CHECK-VF4UF2: call i32 @llvm.vscale.i32() ; CHECK-VF4UF2: %[[VSCALE1:.*]] = call i32 @llvm.vscale.i32() ; CHECK-VF4UF2: %[[MUL1:.*]] = mul i32 %[[VSCALE1]], 4 ; CHECK-VF4UF2: %[[SUB1:.*]] = sub i32 %[[MUL1]], 1 diff --git a/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll b/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll index 034400ca2b0f..2d5db6fc2833 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-inductions.ll @@ -19,33 +19,33 @@ define void @add_ind64_unrolled(ptr noalias nocapture %a, ptr noalias nocapture ; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] -; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv2i64() -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 1 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP6]], i64 0 +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 2 +; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv2i64() +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP8:%.*]] = shl i64 [[TMP7]], 1 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP8]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[STEP_ADD:%.*]] = add [[VEC_IND]], [[DOTSPLAT]] -; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP7]], align 8 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = shl i64 [[TMP8]], 1 -; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP7]], i64 [[TMP9]] -; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP10]], align 8 -; CHECK-NEXT: [[TMP11:%.*]] = add nsw [[WIDE_LOAD]], [[VEC_IND]] -; CHECK-NEXT: [[TMP12:%.*]] = add nsw [[WIDE_LOAD2]], [[STEP_ADD]] -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: store [[TMP11]], ptr [[TMP13]], align 8 -; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl i64 [[TMP14]], 1 -; CHECK-NEXT: [[TMP16:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 [[TMP15]] -; CHECK-NEXT: store [[TMP12]], ptr [[TMP16]], align 8 -; CHECK-NEXT: [[TMP17:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP18:%.*]] = shl i64 [[TMP17]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP18]] +; CHECK-NEXT: [[TMP9:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP9]], align 8 +; CHECK-NEXT: [[TMP10:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP11:%.*]] = shl i64 [[TMP10]], 1 +; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds i64, ptr [[TMP9]], i64 [[TMP11]] +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP12]], align 8 +; CHECK-NEXT: [[TMP13:%.*]] = add nsw [[WIDE_LOAD]], [[VEC_IND]] +; CHECK-NEXT: [[TMP14:%.*]] = add nsw [[WIDE_LOAD2]], [[STEP_ADD]] +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: store [[TMP13]], ptr [[TMP15]], align 8 +; CHECK-NEXT: [[TMP16:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP17:%.*]] = shl i64 [[TMP16]], 1 +; CHECK-NEXT: [[TMP18:%.*]] = getelementptr inbounds i64, ptr [[TMP15]], i64 [[TMP17]] +; CHECK-NEXT: store [[TMP14]], ptr [[TMP18]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[STEP_ADD]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP19:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP19]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -101,30 +101,30 @@ define void @add_ind64_unrolled_nxv1i64(ptr noalias nocapture %a, ptr noalias no ; CHECK-NEXT: [[TMP3:%.*]] = shl i64 [[TMP2]], 1 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] -; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv1i64() -; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP5]], i64 0 +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 1 +; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv1i64() +; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i64 [[TMP7]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP4]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP6]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[STEP_ADD:%.*]] = add [[VEC_IND]], [[DOTSPLAT]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] -; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP6]], align 8 -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[TMP6]], i64 [[TMP7]] -; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP8]], align 8 -; CHECK-NEXT: [[TMP9:%.*]] = add nsw [[WIDE_LOAD]], [[VEC_IND]] -; CHECK-NEXT: [[TMP10:%.*]] = add nsw [[WIDE_LOAD2]], [[STEP_ADD]] -; CHECK-NEXT: [[TMP11:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: store [[TMP9]], ptr [[TMP11]], align 8 -; CHECK-NEXT: [[TMP12:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[TMP11]], i64 [[TMP12]] -; CHECK-NEXT: store [[TMP10]], ptr [[TMP13]], align 8 +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i64, ptr [[B:%.*]], i64 [[INDEX]] +; CHECK-NEXT: [[WIDE_LOAD:%.*]] = load , ptr [[TMP8]], align 8 +; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i64, ptr [[TMP8]], i64 [[TMP9]] +; CHECK-NEXT: [[WIDE_LOAD2:%.*]] = load , ptr [[TMP10]], align 8 +; CHECK-NEXT: [[TMP11:%.*]] = add nsw [[WIDE_LOAD]], [[VEC_IND]] +; CHECK-NEXT: [[TMP12:%.*]] = add nsw [[WIDE_LOAD2]], [[STEP_ADD]] +; CHECK-NEXT: [[TMP13:%.*]] = getelementptr inbounds i64, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: store [[TMP11]], ptr [[TMP13]], align 8 ; CHECK-NEXT: [[TMP14:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP15:%.*]] = shl i64 [[TMP14]], 1 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP15]] +; CHECK-NEXT: [[TMP15:%.*]] = getelementptr inbounds i64, ptr [[TMP13]], i64 [[TMP14]] +; CHECK-NEXT: store [[TMP12]], ptr [[TMP15]], align 8 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[STEP_ADD]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP16:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP16]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP5:![0-9]+]] @@ -189,21 +189,21 @@ define void @add_unique_ind32(ptr noalias nocapture %a, i64 %n) { ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] ; CHECK-NEXT: [[DOTCAST:%.*]] = trunc i64 [[N_VEC]] to i32 ; CHECK-NEXT: [[IND_END:%.*]] = shl i32 [[DOTCAST]], 1 -; CHECK-NEXT: [[TMP4:%.*]] = call @llvm.experimental.stepvector.nxv4i32() -; CHECK-NEXT: [[TMP5:%.*]] = shl [[TMP4]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[TMP6:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP7:%.*]] = shl i32 [[TMP6]], 3 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[TMP7]], i64 0 +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = shl i64 [[TMP4]], 2 +; CHECK-NEXT: [[TMP6:%.*]] = call @llvm.experimental.stepvector.nxv4i32() +; CHECK-NEXT: [[TMP7:%.*]] = shl [[TMP6]], shufflevector ( insertelement ( poison, i32 1, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP9:%.*]] = shl i32 [[TMP8]], 3 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, i32 [[TMP9]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP5]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: store [[VEC_IND]], ptr [[TMP8]], align 4 -; CHECK-NEXT: [[TMP9:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP10:%.*]] = shl i64 [[TMP9]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP10]] +; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[TMP7]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] +; CHECK-NEXT: [[TMP10:%.*]] = getelementptr inbounds i32, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: store [[VEC_IND]], ptr [[TMP10]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = add [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP11:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP11]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP7:![0-9]+]] @@ -267,25 +267,25 @@ define void @add_unique_indf32(ptr noalias nocapture %a, i64 %n) { ; CHECK-NEXT: [[DOTCAST:%.*]] = sitofp i64 [[N_VEC]] to float ; CHECK-NEXT: [[TMP4:%.*]] = fmul float [[DOTCAST]], 2.000000e+00 ; CHECK-NEXT: [[IND_END:%.*]] = fadd float [[TMP4]], 0.000000e+00 -; CHECK-NEXT: [[TMP5:%.*]] = call @llvm.experimental.stepvector.nxv4i32() -; CHECK-NEXT: [[TMP6:%.*]] = uitofp [[TMP5]] to -; CHECK-NEXT: [[TMP7:%.*]] = fmul [[TMP6]], shufflevector ( insertelement ( poison, float 2.000000e+00, i64 0), poison, zeroinitializer) -; CHECK-NEXT: [[INDUCTION:%.*]] = fadd [[TMP7]], zeroinitializer -; CHECK-NEXT: [[TMP8:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP9:%.*]] = shl i32 [[TMP8]], 2 -; CHECK-NEXT: [[TMP10:%.*]] = uitofp i32 [[TMP9]] to float -; CHECK-NEXT: [[TMP11:%.*]] = fmul float [[TMP10]], 2.000000e+00 -; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, float [[TMP11]], i64 0 +; CHECK-NEXT: [[TMP5:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP6:%.*]] = shl i64 [[TMP5]], 2 +; CHECK-NEXT: [[TMP7:%.*]] = call @llvm.experimental.stepvector.nxv4i32() +; CHECK-NEXT: [[TMP8:%.*]] = uitofp [[TMP7]] to +; CHECK-NEXT: [[TMP9:%.*]] = fmul [[TMP8]], shufflevector ( insertelement ( poison, float 2.000000e+00, i64 0), poison, zeroinitializer) +; CHECK-NEXT: [[INDUCTION:%.*]] = fadd [[TMP9]], zeroinitializer +; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP11:%.*]] = shl i32 [[TMP10]], 2 +; CHECK-NEXT: [[TMP12:%.*]] = uitofp i32 [[TMP11]] to float +; CHECK-NEXT: [[TMP13:%.*]] = fmul float [[TMP12]], 2.000000e+00 +; CHECK-NEXT: [[DOTSPLATINSERT:%.*]] = insertelement poison, float [[TMP13]], i64 0 ; CHECK-NEXT: [[DOTSPLAT:%.*]] = shufflevector [[DOTSPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] ; CHECK-NEXT: [[VEC_IND:%.*]] = phi [ [[INDUCTION]], [[VECTOR_PH]] ], [ [[VEC_IND_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP12:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] -; CHECK-NEXT: store [[VEC_IND]], ptr [[TMP12]], align 4 -; CHECK-NEXT: [[TMP13:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP14:%.*]] = shl i64 [[TMP13]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP14]] +; CHECK-NEXT: [[TMP14:%.*]] = getelementptr inbounds float, ptr [[A:%.*]], i64 [[INDEX]] +; CHECK-NEXT: store [[VEC_IND]], ptr [[TMP14]], align 4 +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP6]] ; CHECK-NEXT: [[VEC_IND_NEXT]] = fadd [[VEC_IND]], [[DOTSPLAT]] ; CHECK-NEXT: [[TMP15:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP15]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP9:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll b/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll index 375d2e0ac8af..b14d5a9f4bb5 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-lifetime.ll @@ -20,18 +20,18 @@ define void @test(ptr %d) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 128, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 128, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 ; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4096, ptr [[ARR]]) -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0 -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i32 100, i64 0), poison, zeroinitializer), ptr [[TMP6]], align 8 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0 +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i32 100, i64 0), poison, zeroinitializer), ptr [[TMP8]], align 8 ; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4096, ptr [[ARR]]) -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] ; CHECK: middle.block: @@ -92,18 +92,18 @@ define void @testloopvariant(ptr %d) { ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 2 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 128, [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 128, [[N_MOD_VF]] +; CHECK-NEXT: [[TMP4:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP5:%.*]] = mul i64 [[TMP4]], 2 ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i64 [ 0, [[VECTOR_PH]] ], [ [[INDEX_NEXT:%.*]], [[VECTOR_BODY]] ] -; CHECK-NEXT: [[TMP4:%.*]] = add i64 [[INDEX]], 0 +; CHECK-NEXT: [[TMP6:%.*]] = add i64 [[INDEX]], 0 ; CHECK-NEXT: call void @llvm.lifetime.end.p0(i64 4096, ptr [[ARR]]) -; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[TMP4]] -; CHECK-NEXT: [[TMP6:%.*]] = getelementptr inbounds i32, ptr [[TMP5]], i32 0 -; CHECK-NEXT: store shufflevector ( insertelement ( poison, i32 100, i64 0), poison, zeroinitializer), ptr [[TMP6]], align 8 +; CHECK-NEXT: [[TMP7:%.*]] = getelementptr inbounds i32, ptr [[D]], i64 [[TMP6]] +; CHECK-NEXT: [[TMP8:%.*]] = getelementptr inbounds i32, ptr [[TMP7]], i32 0 +; CHECK-NEXT: store shufflevector ( insertelement ( poison, i32 100, i64 0), poison, zeroinitializer), ptr [[TMP8]], align 8 ; CHECK-NEXT: call void @llvm.lifetime.start.p0(i64 4096, ptr [[ARR]]) -; CHECK-NEXT: [[TMP7:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP8:%.*]] = mul i64 [[TMP7]], 2 -; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP8]] +; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP5]] ; CHECK-NEXT: [[TMP9:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP9]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll b/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll index 6de52c09665c..33248e13320b 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-loop-unpredicated-body-scalar-tail.ll @@ -12,6 +12,8 @@ ; CHECKUF1-DAG: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2 ; CHECKUF1-DAG: %n.mod.vf = urem i64 %wide.trip.count, %[[VSCALEX4]] ; CHECKUF1: %n.vec = sub nsw i64 %wide.trip.count, %n.mod.vf +; CHECKUF1: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64() +; CHECKUF1: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2 ; CHECKUF1: vector.body: ; CHECKUF1: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] @@ -20,8 +22,6 @@ ; CHECKUF1: %[[FADD:.*]] = fadd %wide.load, shufflevector ( insertelement ( poison, double 1.000000e+00, i64 0), poison, zeroinitializer) ; CHECKUF1: %[[IDXA:.*]] = getelementptr inbounds double, ptr %a, i64 %index ; CHECKUF1: store %[[FADD]], ptr %[[IDXA]], align 8 -; CHECKUF1: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64() -; CHECKUF1: %[[VSCALEX4:.*]] = shl i64 %[[VSCALE]], 2 ; CHECKUF1: %index.next = add nuw i64 %index, %[[VSCALEX4]] ; CHECKUF1: %[[CMP:.*]] = icmp eq i64 %index.next, %n.vec ; CHECKUF1: br i1 %[[CMP]], label %middle.block, label %vector.body, !llvm.loop !0 @@ -41,6 +41,8 @@ ; CHECKUF2-DAG: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3 ; CHECKUF2-DAG: %n.mod.vf = urem i64 %wide.trip.count, %[[VSCALEX8]] ; CHECKUF2: %n.vec = sub nsw i64 %wide.trip.count, %n.mod.vf +; CHECKUF2: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64() +; CHECKUF2: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3 ; CHECKUF2: vector.body: ; CHECKUF2: %index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ] @@ -58,8 +60,6 @@ ; CHECKUF2: %[[VSCALE2:.*]] = shl i64 %[[VSCALE]], 2 ; CHECKUF2: %[[IDXA_NEXT:.*]] = getelementptr inbounds double, ptr %[[IDXA]], i64 %[[VSCALE2]] ; CHECKUF2: store %[[FADD_NEXT]], ptr %[[IDXA_NEXT]], align 8 -; CHECKUF2: %[[VSCALE:.*]] = call i64 @llvm.vscale.i64() -; CHECKUF2: %[[VSCALEX8:.*]] = shl i64 %[[VSCALE]], 3 ; CHECKUF2: %index.next = add nuw i64 %index, %[[VSCALEX8]] ; CHECKUF2: %[[CMP:.*]] = icmp eq i64 %index.next, %n.vec ; CHECKUF2: br i1 %[[CMP]], label %middle.block, label %vector.body, !llvm.loop !0 diff --git a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll index afe16c71f7f9..e68e658f0e87 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-reduction-inloop.ll @@ -5,6 +5,10 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 define i8 @reduction_add_trunc(ptr noalias nocapture %A) { ; CHECK-LABEL: @reduction_add_trunc( +; CHECK: call i32 @llvm.vscale.i32() +; CHECK: call i32 @llvm.vscale.i32() +; CHECK: [[TMP30:%.*]] = call i32 @llvm.vscale.i32() +; CHECK-NEXT: [[TMP31:%.*]] = mul i32 [[TMP30]], 16 ; CHECK: vector.body: ; CHECK-NEXT: [[INDEX:%.*]] = phi i32 [ 0, %vector.ph ], [ [[INDEX_NEXT:%.*]], %vector.body ] ; CHECK-NEXT: [[VEC_PHI:%.*]] = phi [ insertelement ( zeroinitializer, i32 255, i32 0), %vector.ph ], [ [[TMP34:%.*]], %vector.body ] @@ -21,8 +25,6 @@ define i8 @reduction_add_trunc(ptr noalias nocapture %A) { ; CHECK-NEXT: [[TMP35:%.*]] = trunc [[TMP29]] to ; CHECK-NEXT: [[TMP34]] = zext [[TMP33]] to ; CHECK-NEXT: [[TMP36]] = zext [[TMP35]] to -; CHECK-NEXT: [[TMP30:%.*]] = call i32 @llvm.vscale.i32() -; CHECK-NEXT: [[TMP31:%.*]] = mul i32 [[TMP30]], 16 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i32 [[INDEX]], [[TMP31]] ; CHECK-NEXT: [[TMP32:%.*]] = icmp eq i32 [[INDEX_NEXT]], {{%.*}} ; CHECK: middle.block: diff --git a/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll b/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll index 8c92074abfd8..18d2323ed6f5 100644 --- a/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll +++ b/llvm/test/Transforms/LoopVectorize/scalable-trunc-min-bitwidth.ll @@ -13,6 +13,8 @@ define void @trunc_minimal_bitwidth(ptr %bptr, ptr noalias %hptr, i32 %val, i64 ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[N]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[N]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[VAL:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: br label [[VECTOR_BODY:%.*]] @@ -21,8 +23,6 @@ define void @trunc_minimal_bitwidth(ptr %bptr, ptr noalias %hptr, i32 %val, i64 ; CHECK-NEXT: [[TMP4:%.*]] = trunc [[BROADCAST_SPLAT]] to ; CHECK-NEXT: [[TMP5:%.*]] = getelementptr inbounds i16, ptr [[HPTR:%.*]], i64 [[INDEX]] ; CHECK-NEXT: store [[TMP4]], ptr [[TMP5]], align 2 -; CHECK-NEXT: [[TMP6:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP7:%.*]] = mul i64 [[TMP6]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP7]] ; CHECK-NEXT: [[TMP8:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP8]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP0:![0-9]+]] @@ -73,6 +73,8 @@ define void @trunc_minimal_bitwidths_shufflevector (ptr %p, i32 %arg1, i64 %len) ; CHECK-NEXT: [[TMP3:%.*]] = mul i64 [[TMP2]], 4 ; CHECK-NEXT: [[N_MOD_VF:%.*]] = urem i64 [[LEN]], [[TMP3]] ; CHECK-NEXT: [[N_VEC:%.*]] = sub i64 [[LEN]], [[N_MOD_VF]] +; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() +; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; CHECK-NEXT: [[BROADCAST_SPLATINSERT:%.*]] = insertelement poison, i32 [[ARG1:%.*]], i64 0 ; CHECK-NEXT: [[BROADCAST_SPLAT:%.*]] = shufflevector [[BROADCAST_SPLATINSERT]], poison, zeroinitializer ; CHECK-NEXT: [[TMP4:%.*]] = trunc [[BROADCAST_SPLAT]] to @@ -84,8 +86,6 @@ define void @trunc_minimal_bitwidths_shufflevector (ptr %p, i32 %arg1, i64 %len) ; CHECK-NEXT: [[TMP6:%.*]] = xor [[WIDE_LOAD]], [[TMP4]] ; CHECK-NEXT: [[TMP7:%.*]] = mul [[TMP6]], [[WIDE_LOAD]] ; CHECK-NEXT: store [[TMP7]], ptr [[TMP5]], align 1 -; CHECK-NEXT: [[TMP8:%.*]] = call i64 @llvm.vscale.i64() -; CHECK-NEXT: [[TMP9:%.*]] = mul i64 [[TMP8]], 4 ; CHECK-NEXT: [[INDEX_NEXT]] = add nuw i64 [[INDEX]], [[TMP9]] ; CHECK-NEXT: [[TMP10:%.*]] = icmp eq i64 [[INDEX_NEXT]], [[N_VEC]] ; CHECK-NEXT: br i1 [[TMP10]], label [[MIDDLE_BLOCK:%.*]], label [[VECTOR_BODY]], !llvm.loop [[LOOP4:![0-9]+]] diff --git a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll index 9a4f174f439a..305fb7f8ab8d 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-dot-printing.ll @@ -8,7 +8,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounwind uwtable { ; CHECK: digraph VPlan { -; CHECK-NEXT: graph [labelloc=t, fontsize=30; label="Vectorization Plan\nInitial VPlan for VF=\{4\},UF\>=1\nLive-in vp\<[[VEC_TC:%.+]]\> = vector-trip-count\nLive-in ir\<%n\> = original trip-count\n"] +; CHECK-NEXT: graph [labelloc=t, fontsize=30; label="Vectorization Plan\nInitial VPlan for VF=\{4\},UF\>=1\nLive-in vp\<[[VFxUF:%.+]]\> = VF * UF\nLive-in vp\<[[VEC_TC:%.+]]\> = vector-trip-count\nLive-in ir\<%n\> = original trip-count\n"] ; CHECK-NEXT: node [shape=rect, fontname=Courier, fontsize=30] ; CHECK-NEXT: edge [fontname=Courier, fontsize=30] ; CHECK-NEXT: compound=true @@ -33,7 +33,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: " WIDEN-CALL ir\<%call\> = call @llvm.sqrt.f32(ir\<%lv\>) (using vector intrinsic)\l" + ; CHECK-NEXT: " CLONE ir\<%arrayidx2\> = getelementptr inbounds ir\<%x\>, vp\<[[STEPS]]\>\l" + ; CHECK-NEXT: " WIDEN store ir\<%arrayidx2\>, ir\<%call\>\l" + -; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT]]\> = VF * UF + nuw vp\<[[CAN_IV]]\>\l" + +; CHECK-NEXT: " EMIT vp\<[[CAN_IV_NEXT]]\> = add nuw vp\<[[CAN_IV]]\>, vp\<[[VFxUF]]\>\l" + ; CHECK-NEXT: " EMIT branch-on-count vp\<[[CAN_IV_NEXT]]\>, vp\<[[VEC_TC]]\>\l" + ; CHECK-NEXT: "No successors\l" ; CHECK-NEXT: ] diff --git a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll index 70ede8dc12b9..8d50993faf9e 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-iv-transforms.ll @@ -4,6 +4,7 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { ; CHECK-LABEL: LV: Checking a loop in 'iv_no_binary_op_in_descriptor' ; CHECK: VPlan 'Initial VPlan for VF={8},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: @@ -17,7 +18,7 @@ define void @iv_no_binary_op_in_descriptor(i1 %c, ptr %dst) { ; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> ; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%dst>, vp<[[STEPS:%.+]]> ; CHECK-NEXT: WIDEN store ir<%gep>, ir<%iv> -; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_INC:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_INC]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll index f129cf3e2a1a..13f444eaaf76 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-printing.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-printing.ll @@ -9,6 +9,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounwind uwtable { ; CHECK-LABEL: Checking a loop in 'print_call_and_memory' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -24,7 +25,7 @@ define void @print_call_and_memory(i64 %n, ptr noalias %y, ptr noalias %x) nounw ; CHECK-NEXT: WIDEN-CALL ir<%call> = call @llvm.sqrt.f32(ir<%lv>) ; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%call> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -56,6 +57,7 @@ for.end: ; preds = %for.body, %entry define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x, ptr %z) nounwind uwtable { ; CHECK-LABEL: Checking a loop in 'print_widen_gep_and_select' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -74,7 +76,7 @@ define void @print_widen_gep_and_select(i64 %n, ptr noalias %y, ptr noalias %x, ; CHECK-NEXT: WIDEN ir<%add> = fadd ir<%lv>, ir<%sel> ; CHECK-NEXT: CLONE ir<%arrayidx2> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%arrayidx2>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -108,6 +110,7 @@ for.end: ; preds = %for.body, %entry define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-LABEL: Checking a loop in 'print_reduction' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -122,7 +125,7 @@ define float @print_reduction(i64 %n, ptr noalias %y) { ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> ; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -154,6 +157,7 @@ for.end: ; preds = %for.body, %entry define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr noalias %dst) { ; CHECK-LABEL: Checking a loop in 'print_reduction_with_invariant_store' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -168,7 +172,7 @@ define void @print_reduction_with_invariant_store(i64 %n, ptr noalias %y, ptr no ; CHECK-NEXT: CLONE ir<%arrayidx> = getelementptr inbounds ir<%y>, vp<[[IV]]> ; CHECK-NEXT: WIDEN ir<%lv> = load ir<%arrayidx> ; CHECK-NEXT: REDUCE ir<%red.next> = ir<%red> + fast reduce.fadd (ir<%lv>) (with final reduction value stored in invariant address sank outside of loop) -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -199,6 +203,7 @@ for.end: ; preds = %for.body, %entry define void @print_replicate_predicated_phi(i64 %n, ptr %x) { ; CHECK-LABEL: Checking a loop in 'print_replicate_predicated_phi' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count ; CHECK-EMPTY: @@ -237,7 +242,7 @@ define void @print_replicate_predicated_phi(i64 %n, ptr %x) { ; CHECK-NEXT: BLEND ir<%d> = ir<0>/vp<[[NOT]]> vp<[[PRED]]>/ir<%cmp> ; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%x>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%idx>, ir<%d> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -277,6 +282,7 @@ for.end: ; preds = %for.inc define void @print_interleave_groups(i32 %C, i32 %D) { ; CHECK-LABEL: Checking a loop in 'print_interleave_groups' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<256> = original trip-count ; CHECK-EMPTY: @@ -301,7 +307,7 @@ define void @print_interleave_groups(i32 %C, i32 %D) { ; CHECK-NEXT: store ir<1> to index 1 ; CHECK-NEXT: store ir<2> to index 2 ; CHECK-NEXT: store ir<%AB.3> to index 3 -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -345,6 +351,7 @@ for.end: define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-LABEL: Checking a loop in 'print_fmuladd_strict' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -362,7 +369,7 @@ define float @print_fmuladd_strict(ptr %a, ptr %b, i64 %n) { ; CHECK-NEXT: WIDEN ir<%l.b> = load ir<%arrayidx2> ; CHECK-NEXT: EMIT vp<[[FMUL:%.+]]> = fmul nnan ninf nsz ir<%l.a>, ir<%l.b> ; CHECK-NEXT: REDUCE ir<[[MULADD:%.+]]> = ir<%sum.07> + nnan ninf nsz reduce.fadd (vp<[[FMUL]]>) -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -396,6 +403,7 @@ for.end: define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !dbg !5 { ; CHECK-LABEL: Checking a loop in 'debug_loc_vpinstruction' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<128> = original trip-count ; CHECK-EMPTY: @@ -436,7 +444,7 @@ define void @debug_loc_vpinstruction(ptr nocapture %asd, ptr nocapture %bsd) !db ; CHECK-NEXT: EMIT vp<[[SEL2:%.+]]> = select vp<[[NOT1]]>, vp<[[NOT2]]>, ir ; CHECK-NEXT: BLEND ir<%ysd.0> = vp<[[PHI]]>/vp<[[OR1]]> ir<%psd>/vp<[[SEL2]]> ; CHECK-NEXT: WIDEN store ir<%isd>, ir<%ysd.0> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT:} @@ -482,6 +490,7 @@ declare float @llvm.fmuladd.f32(float, float, float) define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-LABEL: Checking a loop in 'print_expand_scev' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VTC:%.+]]> = vector-trip-count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count ; CHECK-EMPTY: @@ -504,7 +513,7 @@ define void @print_expand_scev(i64 %y, ptr %ptr) { ; CHECK-NEXT: WIDEN ir<%v3> = add nuw ir<%v2>, ir<1> ; CHECK-NEXT: REPLICATE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> ; CHECK-NEXT: REPLICATE store ir<%v3>, ir<%gep> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VTC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -537,6 +546,7 @@ loop.exit: define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-LABEL: Checking a loop in 'print_exit_value' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: @@ -551,7 +561,7 @@ define i32 @print_exit_value(ptr %ptr, i32 %off) { ; CHECK-NEXT: CLONE ir<%gep> = getelementptr inbounds ir<%ptr>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN ir<%add> = add ir<%iv>, ir<%off> ; CHECK-NEXT: WIDEN store ir<%gep>, ir<0> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -583,6 +593,7 @@ exit: define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr %z) { ; CHECK-LABEL: Checking a loop in 'print_fast_math_flags' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -600,7 +611,7 @@ define void @print_fast_math_flags(i64 %n, ptr noalias %y, ptr noalias %x, ptr % ; CHECK-NEXT: WIDEN ir<%div> = fdiv reassoc nsz contract ir<%mul>, ir<2.000000e+00> ; CHECK-NEXT: CLONE ir<%gep.x> = getelementptr inbounds ir<%x>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%gep.x>, ir<%div> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -633,6 +644,7 @@ exit: define void @print_exact_flags(i64 %n, ptr noalias %x) { ; CHECK-LABEL: Checking a loop in 'print_exact_flags' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -649,7 +661,7 @@ define void @print_exact_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: WIDEN ir<%div.2> = udiv ir<%lv>, ir<60> ; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%div.1>, ir<%div.2> ; CHECK-NEXT: WIDEN store ir<%gep.x>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -681,6 +693,7 @@ exit: define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) { ; CHECK-LABEL: Checking a loop in 'print_call_flags' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -719,7 +732,7 @@ define void @print_call_flags(ptr readonly %src, ptr noalias %dest, i64 %n) { ; CHECK-NEXT: BLEND ir<%st.value> = ir<%ld.value>/vp<[[NOT_COND]]> ir<%fadd>/ir<%ifcond> ; CHECK-NEXT: CLONE ir<%st.addr> = getelementptr inbounds ir<%dest>, vp<[[STEPS]]> ; CHECK-NEXT: WIDEN store ir<%st.addr>, ir<%st.value> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -761,6 +774,7 @@ end: define void @print_disjoint_flags(i64 %n, ptr noalias %x) { ; CHECK-LABEL: Checking a loop in 'print_disjoint_flags' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<%n> = original trip-count ; CHECK-EMPTY: @@ -777,7 +791,7 @@ define void @print_disjoint_flags(i64 %n, ptr noalias %x) { ; CHECK-NEXT: WIDEN ir<%or.2> = or ir<%lv>, ir<3> ; CHECK-NEXT: WIDEN ir<%add> = add nuw nsw ir<%or.1>, ir<%or.2> ; CHECK-NEXT: WIDEN store ir<%gep.x>, ir<%add> -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -809,23 +823,23 @@ exit: define void @zext_nneg(ptr noalias %p, ptr noalias %p1) { ; CHECK-LABEL: LV: Checking a loop in 'zext_nneg' ; CHECK: VPlan 'Initial VPlan for VF={4},UF>=1' { -; CHECK-NEXT: Live-in vp<%0> = vector-trip-count -; CHECK-NEXT: Live-in ir<0> = original trip-count +; CHECK-NEXT: Live-in vp<[[VFxUF:%.]]> = VF * UF +; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count +; CHECK-NEXT: Live-in ir<1000> = original trip-count ; CHECK-EMPTY: ; CHECK-NEXT: vector.ph: ; CHECK-NEXT: Successor(s): vector loop ; CHECK-EMPTY: ; CHECK-NEXT: vector loop: { ; CHECK-NEXT: vector.body: -; CHECK-NEXT: EMIT vp<%1> = CANONICAL-INDUCTION ir<0>, vp<%8> -; CHECK-NEXT: vp<%2> = DERIVED-IV ir<0> + vp<%1> * ir<1> (truncated to i32) -; CHECK-NEXT: vp<%3> = SCALAR-STEPS vp<%2>, ir<1> -; CHECK-NEXT: CLONE ir<%zext> = zext nneg vp<%3> -; CHECK-NEXT: CLONE ir<%idx2> = getelementptr ir<%p>, ir<%zext> -; CHECK-NEXT: WIDEN ir<%1> = load ir<%idx2> -; CHECK-NEXT: REPLICATE store ir<%1>, ir<%p1> -; CHECK-NEXT: EMIT vp<%8> = VF * UF + nuw vp<%1> -; CHECK-NEXT: EMIT branch-on-count vp<%8>, vp<%0> +; CHECK-NEXT: EMIT vp<[[CAN_IV:%.+]]> = CANONICAL-INDUCTION ir<0>, vp<[[CAN_IV_NEXT:%.+]]> +; CHECK-NEXT: vp<[[STEPS:%.+]]> = SCALAR-STEPS vp<[[CAN_IV]]>, ir<1> +; CHECK-NEXT: CLONE ir<%idx> = getelementptr ir<%p>, vp<[[STEPS]]> +; CHECK-NEXT: WIDEN ir<%l> = load ir<%idx> +; CHECK-NEXT: WIDEN-CAST ir<%zext> = zext nneg ir<%l> +; CHECK-NEXT: REPLICATE store ir<%zext>, ir<%p1> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT]]> = add nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } ; @@ -834,13 +848,12 @@ entry: body: %iv = phi i64 [ %next, %body ], [ 0, %entry ] - %0 = trunc i64 %iv to i32 - %zext = zext nneg i32 %0 to i64 - %idx2 = getelementptr double, ptr %p, i64 %zext - %1 = load double, ptr %idx2, align 8 - store double %1, ptr %p1, align 8 + %idx = getelementptr i32, ptr %p, i64 %iv + %l = load i32, ptr %idx, align 8 + %zext = zext nneg i32 %l to i64 + store i64 %zext, ptr %p1, align 8 %next = add i64 %iv, 1 - %cmp = icmp eq i64 %next, 0 + %cmp = icmp eq i64 %next, 1000 br i1 %cmp, label %exit, label %body exit: diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll index 0dac63d54557..8c18c107b39f 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge-vf1.ll @@ -8,6 +8,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-LABEL: sink_with_sideeffects ; CHECK: VPlan 'Initial VPlan for VF={1},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: ir<0> = original trip-count ; CHECK-EMPTY: @@ -37,7 +38,7 @@ define void @sink_with_sideeffects(i1 %c, ptr %ptr) { ; CHECK-NEXT: } ; CHECK: if.then.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll index afc46de9a5a1..f2e6dd43a2ec 100644 --- a/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll +++ b/llvm/test/Transforms/LoopVectorize/vplan-sink-scalars-and-merge.ll @@ -11,6 +11,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; CHECK-LABEL: LV: Checking a loop in 'sink1' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -50,7 +51,7 @@ target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f3 ; CHECK-NEXT: } ; CHECK: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -79,6 +80,7 @@ exit: ; CHECK-LABEL: LV: Checking a loop in 'sink2' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -133,7 +135,7 @@ exit: ; CHECK-NEXT: } ; CHECK: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -162,6 +164,7 @@ exit: ; CHECK-LABEL: LV: Checking a loop in 'sink3' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -216,7 +219,7 @@ exit: ; CHECK-NEXT: } ; CHECK: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -247,6 +250,7 @@ exit: define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) { ; CHECK-LABEL: LV: Checking a loop in 'uniform_gep' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<11> = original trip-count @@ -285,7 +289,7 @@ define void @uniform_gep(i64 %k, ptr noalias %A, ptr noalias %B) { ; CHECK-NEXT: Successor(s): loop.then.0 ; CHECK-EMPTY: ; CHECK-NEXT: loop.then.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -317,6 +321,7 @@ exit: define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-LABEL: LV: Checking a loop in 'pred_cfg1' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -378,7 +383,7 @@ define void @pred_cfg1(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): next.0.1 ; CHECK-EMPTY: ; CHECK-NEXT: next.0.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -416,6 +421,7 @@ exit: define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-LABEL: LV: Checking a loop in 'pred_cfg2' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -479,7 +485,7 @@ define void @pred_cfg2(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): then.1.1 ; CHECK-EMPTY: ; CHECK-NEXT: then.1.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -524,6 +530,7 @@ exit: define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-LABEL: LV: Checking a loop in 'pred_cfg3' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -588,7 +595,7 @@ define void @pred_cfg3(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): then.1.2 ; CHECK-EMPTY: ; CHECK-NEXT: then.1.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -632,6 +639,7 @@ exit: define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-LABEL: LV: Checking a loop in 'merge_3_replicate_region' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -695,7 +703,7 @@ define void @merge_3_replicate_region(i32 %k, i32 %j) { ; CHECK-NEXT: Successor(s): then.0.4 ; CHECK-EMPTY: ; CHECK-NEXT: then.0.4: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -736,6 +744,7 @@ exit: define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) { ; CHECK-LABEL: LV: Checking a loop in 'update_2_uses_in_same_recipe_in_merged_block' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -775,7 +784,7 @@ define void @update_2_uses_in_same_recipe_in_merged_block(i32 %k) { ; CHECK-NEXT: Successor(s): loop.2 ; CHECK-EMPTY: ; CHECK-NEXT: loop.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -802,6 +811,7 @@ exit: define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-LABEL: LV: Checking a loop in 'recipe_in_merge_candidate_used_by_first_order_recurrence' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count @@ -859,7 +869,7 @@ define void @recipe_in_merge_candidate_used_by_first_order_recurrence(i32 %k) { ; CHECK-NEXT: Successor(s): loop.2 ; CHECK-EMPTY: ; CHECK-NEXT: loop.2: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -887,6 +897,7 @@ exit: define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) { ; CHECK-LABEL: LV: Checking a loop in 'update_multiple_users' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<999> = original trip-count ; CHECK-EMPTY: @@ -918,7 +929,7 @@ define void @update_multiple_users(ptr noalias %src, ptr noalias %dst, i1 %c) { ; CHECK-NEXT: Successor(s): loop.then.1 ; CHECK-EMPTY: ; CHECK-NEXT: loop.then.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -951,6 +962,7 @@ exit: define void @sinking_requires_duplication(ptr %addr) { ; CHECK-LABEL: LV: Checking a loop in 'sinking_requires_duplication' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in ir<201> = original trip-count ; CHECK-EMPTY: @@ -983,7 +995,7 @@ define void @sinking_requires_duplication(ptr %addr) { ; CHECK-NEXT: Successor(s): then.0 ; CHECK-EMPTY: ; CHECK-NEXT: then.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -1019,6 +1031,7 @@ exit: define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr noalias %dst) optsize { ; CHECK-LABEL: LV: Checking a loop in 'merge_with_dead_gep_between_regions' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: Live-in vp<[[BTC:%.+]]> = backedge-taken count ; CHECK-NEXT: Live-in ir<%n> = original trip-count @@ -1054,7 +1067,7 @@ define void @merge_with_dead_gep_between_regions(i32 %n, ptr noalias %src, ptr n ; CHECK-NEXT: Successor(s): loop.1 ; CHECK-EMPTY: ; CHECK-NEXT: loop.1: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } @@ -1085,6 +1098,7 @@ exit: define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-LABEL: LV: Checking a loop in 'ptr_induction_remove_dead_recipe' ; CHECK: VPlan 'Initial VPlan for VF={2},UF>=1' { +; CHECK-NEXT: Live-in vp<[[VFxUF:%.+]]> = VF * UF ; CHECK-NEXT: Live-in vp<[[VEC_TC:%.+]]> = vector-trip-count ; CHECK-NEXT: vp<[[TC:%.+]]> = original trip-count ; CHECK-EMPTY: @@ -1121,7 +1135,7 @@ define void @ptr_induction_remove_dead_recipe(ptr %start, ptr %end) { ; CHECK-NEXT: Successor(s): if.then.0 ; CHECK-EMPTY: ; CHECK-NEXT: if.then.0: -; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = VF * UF + nuw vp<[[CAN_IV]]> +; CHECK-NEXT: EMIT vp<[[CAN_IV_NEXT:%.+]]> = add nuw vp<[[CAN_IV]]>, vp<[[VFxUF]]> ; CHECK-NEXT: EMIT branch-on-count vp<[[CAN_IV_NEXT]]>, vp<[[VEC_TC]]> ; CHECK-NEXT: No successors ; CHECK-NEXT: } diff --git a/llvm/test/Transforms/MemCpyOpt/pr75010.ll b/llvm/test/Transforms/MemCpyOpt/pr75010.ll new file mode 100644 index 000000000000..c9af7543b42e --- /dev/null +++ b/llvm/test/Transforms/MemCpyOpt/pr75010.ll @@ -0,0 +1,19 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; RUN: opt -S -passes=memcpyopt < %s | FileCheck %s + +define void @f(ptr nocapture noundef writeonly %r, %x) { +; CHECK-LABEL: @f( +; CHECK-NEXT: entry: +; CHECK-NEXT: [[X_ADDR:%.*]] = alloca , align 8 +; CHECK-NEXT: store [[X:%.*]], ptr [[X_ADDR]], align 8 +; CHECK-NEXT: [[TMP0:%.*]] = load i64, ptr [[X_ADDR]], align 8 +; CHECK-NEXT: store i64 [[TMP0]], ptr [[R:%.*]], align 1 +; CHECK-NEXT: ret void +; +entry: + %x.addr = alloca , align 8 + store %x, ptr %x.addr, align 8 + %0 = load i64, ptr %x.addr, align 8 + store i64 %0, ptr %r, align 1 + ret void +} diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll b/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll index e5582548447a..2510aeaebcf0 100644 --- a/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll +++ b/llvm/test/Transforms/PhaseOrdering/X86/vdiv.ll @@ -64,31 +64,31 @@ define void @vdiv(ptr %x, ptr %y, double %a, i32 %N) #0 { ; CHECK-NEXT: br i1 [[CMP_N]], label [[FOR_END]], label [[FOR_BODY_PREHEADER9]] ; CHECK: for.body.preheader9: ; CHECK-NEXT: [[INDVARS_IV_PH:%.*]] = phi i64 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[N_VEC]], [[MIDDLE_BLOCK]] ] -; CHECK-NEXT: [[TMP18:%.*]] = xor i64 [[INDVARS_IV_PH]], -1 -; CHECK-NEXT: [[TMP19:%.*]] = add nsw i64 [[TMP18]], [[WIDE_TRIP_COUNT]] ; CHECK-NEXT: [[XTRAITER:%.*]] = and i64 [[WIDE_TRIP_COUNT]], 7 ; CHECK-NEXT: [[LCMP_MOD_NOT:%.*]] = icmp eq i64 [[XTRAITER]], 0 ; CHECK-NEXT: br i1 [[LCMP_MOD_NOT]], label [[FOR_BODY_PROL_LOOPEXIT:%.*]], label [[FOR_BODY_PROL_PREHEADER:%.*]] ; CHECK: for.body.prol.preheader: -; CHECK-NEXT: [[TMP20:%.*]] = fdiv fast double 1.000000e+00, [[A]] +; CHECK-NEXT: [[TMP18:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: br label [[FOR_BODY_PROL:%.*]] ; CHECK: for.body.prol: ; CHECK-NEXT: [[INDVARS_IV_PROL:%.*]] = phi i64 [ [[INDVARS_IV_NEXT_PROL:%.*]], [[FOR_BODY_PROL]] ], [ [[INDVARS_IV_PH]], [[FOR_BODY_PROL_PREHEADER]] ] ; CHECK-NEXT: [[PROL_ITER:%.*]] = phi i64 [ [[PROL_ITER_NEXT:%.*]], [[FOR_BODY_PROL]] ], [ 0, [[FOR_BODY_PROL_PREHEADER]] ] ; CHECK-NEXT: [[ARRAYIDX_PROL:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_PROL]] ; CHECK-NEXT: [[T0_PROL:%.*]] = load double, ptr [[ARRAYIDX_PROL]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP21:%.*]] = fmul fast double [[T0_PROL]], [[TMP20]] +; CHECK-NEXT: [[TMP19:%.*]] = fmul fast double [[T0_PROL]], [[TMP18]] ; CHECK-NEXT: [[ARRAYIDX2_PROL:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_PROL]] -; CHECK-NEXT: store double [[TMP21]], ptr [[ARRAYIDX2_PROL]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP19]], ptr [[ARRAYIDX2_PROL]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_PROL]] = add nuw nsw i64 [[INDVARS_IV_PROL]], 1 ; CHECK-NEXT: [[PROL_ITER_NEXT]] = add i64 [[PROL_ITER]], 1 ; CHECK-NEXT: [[PROL_ITER_CMP_NOT:%.*]] = icmp eq i64 [[PROL_ITER_NEXT]], [[XTRAITER]] ; CHECK-NEXT: br i1 [[PROL_ITER_CMP_NOT]], label [[FOR_BODY_PROL_LOOPEXIT]], label [[FOR_BODY_PROL]], !llvm.loop [[LOOP10:![0-9]+]] ; CHECK: for.body.prol.loopexit: ; CHECK-NEXT: [[INDVARS_IV_UNR:%.*]] = phi i64 [ [[INDVARS_IV_PH]], [[FOR_BODY_PREHEADER9]] ], [ [[INDVARS_IV_NEXT_PROL]], [[FOR_BODY_PROL]] ] -; CHECK-NEXT: [[TMP22:%.*]] = icmp ult i64 [[TMP19]], 7 -; CHECK-NEXT: br i1 [[TMP22]], label [[FOR_END]], label [[FOR_BODY_PREHEADER9_NEW:%.*]] +; CHECK-NEXT: [[TMP20:%.*]] = sub nsw i64 [[INDVARS_IV_PH]], [[WIDE_TRIP_COUNT]] +; CHECK-NEXT: [[TMP21:%.*]] = icmp ugt i64 [[TMP20]], -8 +; CHECK-NEXT: br i1 [[TMP21]], label [[FOR_END]], label [[FOR_BODY_PREHEADER9_NEW:%.*]] ; CHECK: for.body.preheader9.new: +; CHECK-NEXT: [[TMP22:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: [[TMP23:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: [[TMP24:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: [[TMP25:%.*]] = fdiv fast double 1.000000e+00, [[A]] @@ -96,57 +96,56 @@ define void @vdiv(ptr %x, ptr %y, double %a, i32 %N) #0 { ; CHECK-NEXT: [[TMP27:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: [[TMP28:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: [[TMP29:%.*]] = fdiv fast double 1.000000e+00, [[A]] -; CHECK-NEXT: [[TMP30:%.*]] = fdiv fast double 1.000000e+00, [[A]] ; CHECK-NEXT: br label [[FOR_BODY:%.*]] ; CHECK: for.body: ; CHECK-NEXT: [[INDVARS_IV:%.*]] = phi i64 [ [[INDVARS_IV_UNR]], [[FOR_BODY_PREHEADER9_NEW]] ], [ [[INDVARS_IV_NEXT_7:%.*]], [[FOR_BODY]] ] ; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV]] ; CHECK-NEXT: [[T0:%.*]] = load double, ptr [[ARRAYIDX]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP31:%.*]] = fmul fast double [[T0]], [[TMP23]] +; CHECK-NEXT: [[TMP30:%.*]] = fmul fast double [[T0]], [[TMP22]] ; CHECK-NEXT: [[ARRAYIDX2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV]] -; CHECK-NEXT: store double [[TMP31]], ptr [[ARRAYIDX2]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP30]], ptr [[ARRAYIDX2]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 1 ; CHECK-NEXT: [[ARRAYIDX_1:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT]] ; CHECK-NEXT: [[T0_1:%.*]] = load double, ptr [[ARRAYIDX_1]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP32:%.*]] = fmul fast double [[T0_1]], [[TMP24]] +; CHECK-NEXT: [[TMP31:%.*]] = fmul fast double [[T0_1]], [[TMP23]] ; CHECK-NEXT: [[ARRAYIDX2_1:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT]] -; CHECK-NEXT: store double [[TMP32]], ptr [[ARRAYIDX2_1]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP31]], ptr [[ARRAYIDX2_1]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_1:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 2 ; CHECK-NEXT: [[ARRAYIDX_2:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_1]] ; CHECK-NEXT: [[T0_2:%.*]] = load double, ptr [[ARRAYIDX_2]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP33:%.*]] = fmul fast double [[T0_2]], [[TMP25]] +; CHECK-NEXT: [[TMP32:%.*]] = fmul fast double [[T0_2]], [[TMP24]] ; CHECK-NEXT: [[ARRAYIDX2_2:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_1]] -; CHECK-NEXT: store double [[TMP33]], ptr [[ARRAYIDX2_2]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP32]], ptr [[ARRAYIDX2_2]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_2:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 3 ; CHECK-NEXT: [[ARRAYIDX_3:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_2]] ; CHECK-NEXT: [[T0_3:%.*]] = load double, ptr [[ARRAYIDX_3]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP34:%.*]] = fmul fast double [[T0_3]], [[TMP26]] +; CHECK-NEXT: [[TMP33:%.*]] = fmul fast double [[T0_3]], [[TMP25]] ; CHECK-NEXT: [[ARRAYIDX2_3:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_2]] -; CHECK-NEXT: store double [[TMP34]], ptr [[ARRAYIDX2_3]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP33]], ptr [[ARRAYIDX2_3]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_3:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 4 ; CHECK-NEXT: [[ARRAYIDX_4:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_3]] ; CHECK-NEXT: [[T0_4:%.*]] = load double, ptr [[ARRAYIDX_4]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP35:%.*]] = fmul fast double [[T0_4]], [[TMP27]] +; CHECK-NEXT: [[TMP34:%.*]] = fmul fast double [[T0_4]], [[TMP26]] ; CHECK-NEXT: [[ARRAYIDX2_4:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_3]] -; CHECK-NEXT: store double [[TMP35]], ptr [[ARRAYIDX2_4]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP34]], ptr [[ARRAYIDX2_4]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_4:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 5 ; CHECK-NEXT: [[ARRAYIDX_5:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_4]] ; CHECK-NEXT: [[T0_5:%.*]] = load double, ptr [[ARRAYIDX_5]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP36:%.*]] = fmul fast double [[T0_5]], [[TMP28]] +; CHECK-NEXT: [[TMP35:%.*]] = fmul fast double [[T0_5]], [[TMP27]] ; CHECK-NEXT: [[ARRAYIDX2_5:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_4]] -; CHECK-NEXT: store double [[TMP36]], ptr [[ARRAYIDX2_5]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP35]], ptr [[ARRAYIDX2_5]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_5:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 6 ; CHECK-NEXT: [[ARRAYIDX_6:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_5]] ; CHECK-NEXT: [[T0_6:%.*]] = load double, ptr [[ARRAYIDX_6]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP37:%.*]] = fmul fast double [[T0_6]], [[TMP29]] +; CHECK-NEXT: [[TMP36:%.*]] = fmul fast double [[T0_6]], [[TMP28]] ; CHECK-NEXT: [[ARRAYIDX2_6:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_5]] -; CHECK-NEXT: store double [[TMP37]], ptr [[ARRAYIDX2_6]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP36]], ptr [[ARRAYIDX2_6]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_6:%.*]] = add nuw nsw i64 [[INDVARS_IV]], 7 ; CHECK-NEXT: [[ARRAYIDX_7:%.*]] = getelementptr inbounds double, ptr [[Y]], i64 [[INDVARS_IV_NEXT_6]] ; CHECK-NEXT: [[T0_7:%.*]] = load double, ptr [[ARRAYIDX_7]], align 8, !tbaa [[TBAA3]] -; CHECK-NEXT: [[TMP38:%.*]] = fmul fast double [[T0_7]], [[TMP30]] +; CHECK-NEXT: [[TMP37:%.*]] = fmul fast double [[T0_7]], [[TMP29]] ; CHECK-NEXT: [[ARRAYIDX2_7:%.*]] = getelementptr inbounds double, ptr [[X]], i64 [[INDVARS_IV_NEXT_6]] -; CHECK-NEXT: store double [[TMP38]], ptr [[ARRAYIDX2_7]], align 8, !tbaa [[TBAA3]] +; CHECK-NEXT: store double [[TMP37]], ptr [[ARRAYIDX2_7]], align 8, !tbaa [[TBAA3]] ; CHECK-NEXT: [[INDVARS_IV_NEXT_7]] = add nuw nsw i64 [[INDVARS_IV]], 8 ; CHECK-NEXT: [[EXITCOND_NOT_7:%.*]] = icmp eq i64 [[INDVARS_IV_NEXT_7]], [[WIDE_TRIP_COUNT]] ; CHECK-NEXT: br i1 [[EXITCOND_NOT_7]], label [[FOR_END]], label [[FOR_BODY]], !llvm.loop [[LOOP12:![0-9]+]] diff --git a/llvm/test/Transforms/Reassociate/local-cse.ll b/llvm/test/Transforms/Reassociate/local-cse.ll index 1609cb1b36fd..4d0467e263f5 100644 --- a/llvm/test/Transforms/Reassociate/local-cse.ll +++ b/llvm/test/Transforms/Reassociate/local-cse.ll @@ -26,16 +26,16 @@ define void @chain_spanning_several_blocks(i64 %inv1, i64 %inv2, i64 %inv3, i64 ; LOCAL_CSE-LABEL: define void @chain_spanning_several_blocks ; LOCAL_CSE-SAME: (i64 [[INV1:%.*]], i64 [[INV2:%.*]], i64 [[INV3:%.*]], i64 [[INV4:%.*]], i64 [[INV5:%.*]]) { ; LOCAL_CSE-NEXT: bb1: -; LOCAL_CSE-NEXT: [[CHAIN_A0:%.*]] = add i64 [[INV2]], [[INV1]] +; LOCAL_CSE-NEXT: [[CHAIN_A0:%.*]] = add nuw i64 [[INV2]], [[INV1]] ; LOCAL_CSE-NEXT: br label [[BB2:%.*]] ; LOCAL_CSE: bb2: ; LOCAL_CSE-NEXT: [[VAL_BB2:%.*]] = call i64 @get_val() -; LOCAL_CSE-NEXT: [[CHAIN_A1:%.*]] = add i64 [[CHAIN_A0]], [[INV4]] -; LOCAL_CSE-NEXT: [[CHAIN_A2:%.*]] = add i64 [[CHAIN_A1]], [[VAL_BB2]] -; LOCAL_CSE-NEXT: [[CHAIN_B1:%.*]] = add i64 [[CHAIN_A0]], [[INV5]] -; LOCAL_CSE-NEXT: [[CHAIN_B2:%.*]] = add i64 [[CHAIN_B1]], [[VAL_BB2]] -; LOCAL_CSE-NEXT: [[CHAIN_C0:%.*]] = add i64 [[INV3]], [[INV1]] -; LOCAL_CSE-NEXT: [[CHAIN_C1:%.*]] = add i64 [[CHAIN_C0]], [[VAL_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_A1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV4]] +; LOCAL_CSE-NEXT: [[CHAIN_A2:%.*]] = add nuw i64 [[CHAIN_A1]], [[VAL_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_B1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV5]] +; LOCAL_CSE-NEXT: [[CHAIN_B2:%.*]] = add nuw i64 [[CHAIN_B1]], [[VAL_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_C0:%.*]] = add nuw i64 [[INV3]], [[INV1]] +; LOCAL_CSE-NEXT: [[CHAIN_C1:%.*]] = add nuw i64 [[CHAIN_C0]], [[VAL_BB2]] ; LOCAL_CSE-NEXT: call void @keep_alive(i64 [[CHAIN_A2]]) ; LOCAL_CSE-NEXT: call void @keep_alive(i64 [[CHAIN_B2]]) ; LOCAL_CSE-NEXT: call void @keep_alive(i64 [[CHAIN_C1]]) @@ -47,11 +47,11 @@ define void @chain_spanning_several_blocks(i64 %inv1, i64 %inv2, i64 %inv3, i64 ; CSE-NEXT: br label [[BB2:%.*]] ; CSE: bb2: ; CSE-NEXT: [[VAL_BB2:%.*]] = call i64 @get_val() -; CSE-NEXT: [[CHAIN_A0:%.*]] = add i64 [[VAL_BB2]], [[INV1]] -; CSE-NEXT: [[CHAIN_A1:%.*]] = add i64 [[CHAIN_A0]], [[INV2]] +; CSE-NEXT: [[CHAIN_A0:%.*]] = add nuw i64 [[VAL_BB2]], [[INV1]] +; CSE-NEXT: [[CHAIN_A1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV2]] ; CSE-NEXT: [[CHAIN_A2:%.*]] = add nuw nsw i64 [[CHAIN_A1]], [[INV4]] ; CSE-NEXT: [[CHAIN_B2:%.*]] = add nuw nsw i64 [[CHAIN_A1]], [[INV5]] -; CSE-NEXT: [[CHAIN_C1:%.*]] = add i64 [[CHAIN_A0]], [[INV3]] +; CSE-NEXT: [[CHAIN_C1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV3]] ; CSE-NEXT: call void @keep_alive(i64 [[CHAIN_A2]]) ; CSE-NEXT: call void @keep_alive(i64 [[CHAIN_B2]]) ; CSE-NEXT: call void @keep_alive(i64 [[CHAIN_C1]]) @@ -90,19 +90,19 @@ define void @chain_spanning_several_blocks_no_entry_anchor() { ; LOCAL_CSE-NEXT: br label [[BB1:%.*]] ; LOCAL_CSE: bb1: ; LOCAL_CSE-NEXT: [[INV1_BB1:%.*]] = call i64 @get_val() -; LOCAL_CSE-NEXT: [[CHAIN_A0:%.*]] = add i64 [[INV1_BB1]], [[INV2_BB0]] +; LOCAL_CSE-NEXT: [[CHAIN_A0:%.*]] = add nuw i64 [[INV1_BB1]], [[INV2_BB0]] ; LOCAL_CSE-NEXT: br label [[BB2:%.*]] ; LOCAL_CSE: bb2: ; LOCAL_CSE-NEXT: [[INV3_BB2:%.*]] = call i64 @get_val() ; LOCAL_CSE-NEXT: [[INV4_BB2:%.*]] = call i64 @get_val() ; LOCAL_CSE-NEXT: [[INV5_BB2:%.*]] = call i64 @get_val() ; LOCAL_CSE-NEXT: [[VAL_BB2:%.*]] = call i64 @get_val() -; LOCAL_CSE-NEXT: [[CHAIN_A1:%.*]] = add i64 [[CHAIN_A0]], [[INV4_BB2]] -; LOCAL_CSE-NEXT: [[CHAIN_A2:%.*]] = add i64 [[CHAIN_A1]], [[VAL_BB2]] -; LOCAL_CSE-NEXT: [[CHAIN_B1:%.*]] = add i64 [[CHAIN_A0]], [[INV5_BB2]] -; LOCAL_CSE-NEXT: [[CHAIN_B2:%.*]] = add i64 [[CHAIN_B1]], [[VAL_BB2]] -; LOCAL_CSE-NEXT: [[CHAIN_C0:%.*]] = add i64 [[VAL_BB2]], [[INV1_BB1]] -; LOCAL_CSE-NEXT: [[CHAIN_C1:%.*]] = add i64 [[CHAIN_C0]], [[INV3_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_A1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV4_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_A2:%.*]] = add nuw i64 [[CHAIN_A1]], [[VAL_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_B1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV5_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_B2:%.*]] = add nuw i64 [[CHAIN_B1]], [[VAL_BB2]] +; LOCAL_CSE-NEXT: [[CHAIN_C0:%.*]] = add nuw i64 [[VAL_BB2]], [[INV1_BB1]] +; LOCAL_CSE-NEXT: [[CHAIN_C1:%.*]] = add nuw i64 [[CHAIN_C0]], [[INV3_BB2]] ; LOCAL_CSE-NEXT: call void @keep_alive(i64 [[CHAIN_A2]]) ; LOCAL_CSE-NEXT: call void @keep_alive(i64 [[CHAIN_B2]]) ; LOCAL_CSE-NEXT: call void @keep_alive(i64 [[CHAIN_C1]]) @@ -120,11 +120,11 @@ define void @chain_spanning_several_blocks_no_entry_anchor() { ; CSE-NEXT: [[INV4_BB2:%.*]] = call i64 @get_val() ; CSE-NEXT: [[INV5_BB2:%.*]] = call i64 @get_val() ; CSE-NEXT: [[VAL_BB2:%.*]] = call i64 @get_val() -; CSE-NEXT: [[CHAIN_A0:%.*]] = add i64 [[VAL_BB2]], [[INV1_BB1]] -; CSE-NEXT: [[CHAIN_A1:%.*]] = add i64 [[CHAIN_A0]], [[INV2_BB0]] +; CSE-NEXT: [[CHAIN_A0:%.*]] = add nuw i64 [[VAL_BB2]], [[INV1_BB1]] +; CSE-NEXT: [[CHAIN_A1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV2_BB0]] ; CSE-NEXT: [[CHAIN_A2:%.*]] = add nuw nsw i64 [[CHAIN_A1]], [[INV4_BB2]] ; CSE-NEXT: [[CHAIN_B2:%.*]] = add nuw nsw i64 [[CHAIN_A1]], [[INV5_BB2]] -; CSE-NEXT: [[CHAIN_C1:%.*]] = add i64 [[CHAIN_A0]], [[INV3_BB2]] +; CSE-NEXT: [[CHAIN_C1:%.*]] = add nuw i64 [[CHAIN_A0]], [[INV3_BB2]] ; CSE-NEXT: call void @keep_alive(i64 [[CHAIN_A2]]) ; CSE-NEXT: call void @keep_alive(i64 [[CHAIN_B2]]) ; CSE-NEXT: call void @keep_alive(i64 [[CHAIN_C1]]) diff --git a/llvm/test/Transforms/Reassociate/reassoc-mul-nuw.ll b/llvm/test/Transforms/Reassociate/reassoc-mul-nuw.ll new file mode 100644 index 000000000000..682fad8d222b --- /dev/null +++ b/llvm/test/Transforms/Reassociate/reassoc-mul-nuw.ll @@ -0,0 +1,34 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt < %s -passes=reassociate -S | FileCheck %s + +; We cannot preserve nuw flags for mul +define i4 @nuw_preserve_negative(i4 %a, i4 %b, i4 %c) { +; CHECK-LABEL: define i4 @nuw_preserve_negative( +; CHECK-SAME: i4 [[A:%.*]], i4 [[B:%.*]], i4 [[C:%.*]]) { +; CHECK-NEXT: [[V0:%.*]] = mul i4 [[B]], [[A]] +; CHECK-NEXT: [[V1:%.*]] = mul i4 [[V0]], [[C]] +; CHECK-NEXT: ret i4 [[V1]] +; + %v0 = mul nuw i4 %a, %c + %v1 = mul nuw i4 %v0, %b + ret i4 %v1 +} + +; TODO: we can add nuw flags if we know all operands are non-zero. +define i4 @nuw_preserve_non_zero(i4 %a, i4 %b, i4 %c) { +; CHECK-LABEL: define i4 @nuw_preserve_non_zero( +; CHECK-SAME: i4 [[A:%.*]], i4 [[B:%.*]], i4 [[C:%.*]]) { +; CHECK-NEXT: [[A0:%.*]] = add nuw i4 [[A]], 1 +; CHECK-NEXT: [[B0:%.*]] = add nuw i4 [[B]], 1 +; CHECK-NEXT: [[C0:%.*]] = add nuw i4 [[C]], 1 +; CHECK-NEXT: [[V0:%.*]] = mul i4 [[B0]], [[A0]] +; CHECK-NEXT: [[V1:%.*]] = mul i4 [[V0]], [[C0]] +; CHECK-NEXT: ret i4 [[V1]] +; + %a0 = add nuw i4 %a, 1 + %b0 = add nuw i4 %b, 1 + %c0 = add nuw i4 %c, 1 + %v0 = mul nuw i4 %a0, %c0 + %v1 = mul nuw i4 %v0, %b0 + ret i4 %v1 +} diff --git a/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll b/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll new file mode 100644 index 000000000000..8e0f38222224 --- /dev/null +++ b/llvm/test/Transforms/SLPVectorizer/RISCV/remarks-insert-into-small-vector.ll @@ -0,0 +1,56 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 4 +; RUN: opt -S --passes=slp-vectorizer -mtriple=riscv64-unknown-linux -pass-remarks-output=%t -mattr=+v -slp-threshold=-10 < %s | FileCheck %s +; RUN: FileCheck %s --check-prefix=YAML < %t + +; YAML-LABEL: --- !Passed +; YAML-NEXT: Pass: slp-vectorizer +; YAML-NEXT: Name: StoresVectorized +; YAML-NEXT: Function: test +; YAML-NEXT: Args: +; YAML-NEXT: - String: 'Stores SLP vectorized with cost ' +; YAML-NEXT: - Cost: '9' +; YAML-NEXT: - String: ' and with tree size ' +; YAML-NEXT: - TreeSize: '7' + +define void @test() { +; CHECK-LABEL: define void @test( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: entry: +; CHECK-NEXT: [[TMP0:%.*]] = load float, ptr null, align 4 +; CHECK-NEXT: [[TMP1:%.*]] = load float, ptr null, align 4 +; CHECK-NEXT: [[TMP2:%.*]] = load float, ptr null, align 4 +; CHECK-NEXT: [[V9IDX:%.*]] = getelementptr i8, ptr null, i32 4 +; CHECK-NEXT: [[V14IDX:%.*]] = getelementptr i8, ptr null, i32 8 +; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x float> , float [[TMP1]], i32 0 +; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x float> poison, float [[TMP0]], i32 0 +; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x float> [[TMP4]], float [[TMP2]], i32 1 +; CHECK-NEXT: [[TMP6:%.*]] = fcmp ogt <2 x float> [[TMP3]], [[TMP5]] +; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP6]], i32 0 +; CHECK-NEXT: [[V0_0:%.*]] = select i1 [[TMP7]], float [[TMP0]], float 0.000000e+00 +; CHECK-NEXT: [[TMP8:%.*]] = select <2 x i1> [[TMP6]], <2 x float> [[TMP3]], <2 x float> zeroinitializer +; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i1> [[TMP6]], i32 1 +; CHECK-NEXT: [[V9_0:%.*]] = select i1 [[TMP9]], float [[TMP2]], float 0.000000e+00 +; CHECK-NEXT: store float [[V0_0]], ptr null, align 4 +; CHECK-NEXT: store float [[V9_0]], ptr [[V9IDX]], align 4 +; CHECK-NEXT: store <2 x float> [[TMP8]], ptr [[V14IDX]], align 4 +; CHECK-NEXT: ret void +; +entry: + %0 = load float, ptr null, align 4 + %1 = load float, ptr null, align 4 + %2 = load float, ptr null, align 4 + %cmp.i = fcmp ogt float %1, %0 + %v14.0 = select i1 %cmp.i, float %1, float 0.000000e+00 + %v0.0 = select i1 %cmp.i, float %0, float 0.000000e+00 + %cmp4.i = fcmp ogt float 0.000000e+00, %2 + %v19.0 = select i1 %cmp4.i, float 0.000000e+00, float 0.000000e+00 + %v9.0 = select i1 %cmp4.i, float %2, float 0.000000e+00 + store float %v0.0, ptr null, align 4 + %v9idx = getelementptr i8, ptr null, i32 4 + store float %v9.0, ptr %v9idx, align 4 + %v14idx = getelementptr i8, ptr null, i32 8 + store float %v14.0, ptr %v14idx, align 4 + %v19idx = getelementptr i8, ptr null, i32 12 + store float %v19.0, ptr %v19idx, align 4 + ret void +} diff --git a/llvm/test/tools/llvm-exegesis/X86/latency/memory-annotations-unsupported.s b/llvm/test/tools/llvm-exegesis/X86/latency/memory-annotations-unsupported.s index 5d06a0ceff5c..c483d28adf0e 100644 --- a/llvm/test/tools/llvm-exegesis/X86/latency/memory-annotations-unsupported.s +++ b/llvm/test/tools/llvm-exegesis/X86/latency/memory-annotations-unsupported.s @@ -2,7 +2,7 @@ # RUN: not llvm-exegesis -mtriple=x86_64-unknown-unknown -snippets-file=%s -mode=latency 2>&1 | FileCheck %s -# CHECK: llvm-exegesis error: Memory annotations are only supported in subprocess execution mode +# CHECK: llvm-exegesis error: Memory and snippet address annotations are only supported in subprocess execution mode # LLVM-EXEGESIS-MEM-DEF test1 4096 ff diff --git a/llvm/test/tools/llvm-exegesis/X86/latency/snippet-address-annotations-unsupported.s b/llvm/test/tools/llvm-exegesis/X86/latency/snippet-address-annotations-unsupported.s new file mode 100644 index 000000000000..c38c41d2aed9 --- /dev/null +++ b/llvm/test/tools/llvm-exegesis/X86/latency/snippet-address-annotations-unsupported.s @@ -0,0 +1,9 @@ +# REQUIRES: exegesis-can-measure-latency, x86_64-linux + +# RUN: not llvm-exegesis -mtriple=x86_64-unknown-unknown -snippets-file=%s -mode=latency 2>&1 | FileCheck %s + +# CHECK: llvm-exegesis error: Memory and snippet address annotations are only supported in subprocess execution mode + +# LLVM-EXEGESIS-SNIPPET-ADDRESS 10000 + +movq $0, %rax diff --git a/llvm/test/tools/llvm-exegesis/X86/latency/subprocess-address-annotation.s b/llvm/test/tools/llvm-exegesis/X86/latency/subprocess-address-annotation.s new file mode 100644 index 000000000000..1b104b30bb28 --- /dev/null +++ b/llvm/test/tools/llvm-exegesis/X86/latency/subprocess-address-annotation.s @@ -0,0 +1,26 @@ +# REQUIRES: exegesis-can-measure-latency, x86_64-linux + +# RUN: llvm-exegesis -mtriple=x86_64-unknown-unknown -mode=latency -snippets-file=%s -execution-mode=subprocess | FileCheck %s + +# Check that the code is loaded in at the expected address. + +# LLVM-EXEGESIS-SNIPPET-ADDRESS 20000 +# LLVM-EXEGESIS-DEFREG RAX 0 +# LLVM-EXEGESIS-DEFREG R14 127 +# LLVM-EXEGESIS-DEFREG R15 0 +# LLVM-EXEGESIS-DEFREG RDI 0 + +# Load the instruction pointer and round down to the nearest page as there +# will be some setup code loaded in before this part begins to execute. +lea 0(%rip), %rax +shrq $12, %rax +shlq $12, %rax + +cmpq $0x20000, %rax +cmovneq %r14, %r15 + +movq $60, %rax +movq %r15, %rdi +syscall + +# CHECK-NOT: error: 'Child benchmarking process exited with non-zero exit code: Child process returned with unknown exit code' diff --git a/llvm/test/tools/llvm-objcopy/regex-error.test b/llvm/test/tools/llvm-objcopy/regex-error.test new file mode 100644 index 000000000000..ca9bb2f3b4f9 --- /dev/null +++ b/llvm/test/tools/llvm-objcopy/regex-error.test @@ -0,0 +1,13 @@ +## Test if providing objcopy with an invalid regex generates an error. + +# RUN: yaml2obj %s -o %t + +# RUN: not llvm-objcopy --regex --strip-symbol='[^)\' %t /dev/null 2>&1 | FileCheck %s +# CHECK: error: cannot compile regular expression '[^)\' + +!ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_EXEC + Machine: EM_X86_64 diff --git a/llvm/test/tools/llvm-objdump/ELF/dynamic-section-machine-specific.test b/llvm/test/tools/llvm-objdump/ELF/dynamic-section-machine-specific.test index 20219dd4893b..203c210eb46a 100644 --- a/llvm/test/tools/llvm-objdump/ELF/dynamic-section-machine-specific.test +++ b/llvm/test/tools/llvm-objdump/ELF/dynamic-section-machine-specific.test @@ -268,6 +268,9 @@ ProgramHeaders: # AARCH64: Dynamic Section: # AARCH64-NEXT: AARCH64_BTI_PLT 0x0000000000000001 # AARCH64-NEXT: AARCH64_PAC_PLT 0x0000000000000002 +# AARCH64-NEXT: AARCH64_AUTH_RELR 0x0000000000000003 +# AARCH64-NEXT: AARCH64_AUTH_RELRSZ 0x0000000000000004 +# AARCH64-NEXT: AARCH64_AUTH_RELRENT 0x0000000000000005 --- !ELF FileHeader: @@ -283,6 +286,12 @@ Sections: Value: 1 - Tag: DT_AARCH64_PAC_PLT Value: 2 + - Tag: DT_AARCH64_AUTH_RELR + Value: 3 + - Tag: DT_AARCH64_AUTH_RELRSZ + Value: 4 + - Tag: DT_AARCH64_AUTH_RELRENT + Value: 5 - Tag: DT_NULL Value: 0 ProgramHeaders: diff --git a/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s b/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s new file mode 100644 index 000000000000..f28d92eae857 --- /dev/null +++ b/llvm/test/tools/llvm-readobj/ELF/AArch64/aarch64-feature-pauth.s @@ -0,0 +1,98 @@ +# RUN: rm -rf %t && split-file %s %t && cd %t + +# RUN: llvm-mc -filetype=obj -triple=aarch64-linux-gnu abi-tag.s -o tag.o +# RUN: llvm-mc -filetype=obj -triple=aarch64-linux-gnu abi-tag-short.s -o tag-short.o +# RUN: llvm-mc -filetype=obj -triple=aarch64-linux-gnu abi-tag-long.s -o tag-long.o + +# RUN: llvm-readelf --notes tag.o | FileCheck --check-prefix NORMAL %s +# RUN: llvm-readelf --notes tag-short.o | FileCheck --check-prefix SHORT %s +# RUN: llvm-readelf --notes tag-long.o | FileCheck --check-prefix LONG %s + +# NORMAL: AArch64 PAuth ABI tag: platform 0x2a, version 0x1 +# SHORT: AArch64 PAuth ABI tag: +# LONG: AArch64 PAuth ABI tag: platform 0x2a, version 0x1, additional info 0xEFCDAB8967452301 + +# RUN: llvm-readobj --notes tag.o | FileCheck --check-prefix LLVM-NORMAL %s +# RUN: llvm-readobj --notes tag-short.o | FileCheck --check-prefix LLVM-SHORT %s +# RUN: llvm-readobj --notes tag-long.o | FileCheck --check-prefix LLVM-LONG %s + +# LLVM-SHORT: Notes [ +# LLVM-SHORT-NEXT: NoteSection { +# LLVM-SHORT-NEXT: Name: .note.AARCH64-PAUTH-ABI-tag +# LLVM-SHORT-NEXT: Offset: 0x40 +# LLVM-SHORT-NEXT: Size: 0x1C +# LLVM-SHORT-NEXT: Note { +# LLVM-SHORT-NEXT: Owner: ARM +# LLVM-SHORT-NEXT: Data size: 0xC +# LLVM-SHORT-NEXT: Type: NT_ARM_TYPE_PAUTH_ABI_TAG +# LLVM-SHORT-NEXT: Description data ( +# LLVM-SHORT-NEXT: 0000: 2A000000 00000000 01000000 +# LLVM-SHORT-NEXT: ) +# LLVM-SHORT-NEXT: } +# LLVM-SHORT-NEXT: } +# LLVM-SHORT-NEXT: ] + +# LLVM-NORMAL: Notes [ +# LLVM-NORMAL-NEXT: NoteSection { +# LLVM-NORMAL-NEXT: Name: .note.AARCH64-PAUTH-ABI-tag +# LLVM-NORMAL-NEXT: Offset: 0x40 +# LLVM-NORMAL-NEXT: Size: 0x20 +# LLVM-NORMAL-NEXT: Note { +# LLVM-NORMAL-NEXT: Owner: ARM +# LLVM-NORMAL-NEXT: Data size: 0x10 +# LLVM-NORMAL-NEXT: Type: NT_ARM_TYPE_PAUTH_ABI_TAG +# LLVM-NORMAL-NEXT: Platform: 42 +# LLVM-NORMAL-NEXT: Version: 1 +# LLVM-NORMAL-NEXT: } +# LLVM-NORMAL-NEXT: } +# LLVM-NORMAL-NEXT: ] + +# LLVM-LONG: Notes [ +# LLVM-LONG-NEXT: NoteSection { +# LLVM-LONG-NEXT: Name: .note.AARCH64-PAUTH-ABI-tag +# LLVM-LONG-NEXT: Offset: 0x40 +# LLVM-LONG-NEXT: Size: 0x28 +# LLVM-LONG-NEXT: Note { +# LLVM-LONG-NEXT: Owner: ARM +# LLVM-LONG-NEXT: Data size: 0x18 +# LLVM-LONG-NEXT: Type: NT_ARM_TYPE_PAUTH_ABI_TAG +# LLVM-LONG-NEXT: Platform: 42 +# LLVM-LONG-NEXT: Version: 1 +# LLVM-LONG-NEXT: Additional info: EFCDAB8967452301 +# LLVM-LONG-NEXT: } +# LLVM-LONG-NEXT: } +# LLVM-LONG-NEXT: ] + +#--- abi-tag.s + +.section ".note.AARCH64-PAUTH-ABI-tag", "a" +.long 4 +.long 16 +.long 1 +.asciz "ARM" + +.quad 42 // platform +.quad 1 // version + +#--- abi-tag-short.s + +.section ".note.AARCH64-PAUTH-ABI-tag", "a" +.long 4 +.long 12 +.long 1 +.asciz "ARM" + +.quad 42 +.word 1 + +#--- abi-tag-long.s + +.section ".note.AARCH64-PAUTH-ABI-tag", "a" +.long 4 +.long 24 +.long 1 +.asciz "ARM" + +.quad 42 // platform +.quad 1 // version +.quad 0x0123456789ABCDEF // extra data diff --git a/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test b/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test index 1a1c6dd4d0d1..9f18248e7921 100644 --- a/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test +++ b/llvm/test/tools/llvm-readobj/ELF/broken-dynamic-reloc.test @@ -102,7 +102,7 @@ ProgramHeaders: # RUN: llvm-readelf --dyn-relocations %t3 2>&1 | \ # RUN: FileCheck %s -DFILE=%t3 --check-prefix=INVALID-DT-RELAENT --implicit-check-not=warning: -## INVALID-DT-RELAENT: warning: '[[FILE]]': invalid DT_RELASZ value (0x18) or DT_RELAENT value (0xff) +# INVALID-DT-RELAENT: warning: '[[FILE]]': invalid DT_RELASZ value (0x18) or DT_RELAENT value (0xff) ## Show we print a warning for an invalid relocation table size stored in a DT_RELSZ entry. @@ -112,7 +112,7 @@ ProgramHeaders: # RUN: llvm-readobj --dyn-relocations %t4a 2>&1 | FileCheck %s -DFILE=%t4a --check-prefix=INVALID-DT-RELSZ1 # RUN: llvm-readelf --dyn-relocations %t4a 2>&1 | FileCheck %s -DFILE=%t4a --check-prefix=INVALID-DT-RELSZ1 -## INVALID-DT-RELSZ1: warning: '[[FILE]]': invalid DT_RELSZ value (0xff) or DT_RELENT value (0x18) +# INVALID-DT-RELSZ1: warning: '[[FILE]]': invalid DT_RELSZ value (0xff) or DT_RELENT value (0x18) ## Case B: the DT_RELSZ has value of 0x251, what is too large, because the relocation table goes past the EOF. @@ -127,7 +127,7 @@ ProgramHeaders: # RUN: llvm-readobj --dyn-relocations %t5 2>&1 | FileCheck %s -DFILE=%t5 --check-prefix=INVALID-DT-RELENT # RUN: llvm-readelf --dyn-relocations %t5 2>&1 | FileCheck %s -DFILE=%t5 --check-prefix=INVALID-DT-RELENT -## INVALID-DT-RELENT: warning: '[[FILE]]': invalid DT_RELSZ value (0x18) or DT_RELENT value (0xff) +# INVALID-DT-RELENT: warning: '[[FILE]]': invalid DT_RELSZ value (0x18) or DT_RELENT value (0xff) ## Show we print a warning for an invalid relocation table size stored in a DT_RELRSZ/DT_ANDROID_RELRSZ entry. # RUN: yaml2obj --docnum=2 -DRELTYPE=RELR -DTAG1=DT_RELRSZ -DTAG1VAL=0xFF -DTAG2=DT_RELRENT %s -o %t6 @@ -138,8 +138,8 @@ ProgramHeaders: # RUN: llvm-readobj --dyn-relocations %t7 2>&1 | FileCheck %s -DFILE=%t7 --check-prefix=INVALID-DT-ANDROID-RELRSZ # RUN: llvm-readelf --dyn-relocations %t7 2>&1 | FileCheck %s -DFILE=%t7 --check-prefix=INVALID-DT-ANDROID-RELRSZ -## INVALID-DT-RELRSZ: warning: '[[FILE]]': invalid DT_RELRSZ value (0xff) or DT_RELRENT value (0x18) -## INVALID-DT-ANDROID-RELRSZ: warning: '[[FILE]]': invalid DT_ANDROID_RELRSZ value (0xff) or DT_ANDROID_RELRENT value (0x18) +# INVALID-DT-RELRSZ: warning: '[[FILE]]': invalid DT_RELRSZ value (0xff) or DT_RELRENT value (0x18) +# INVALID-DT-ANDROID-RELRSZ: warning: '[[FILE]]': invalid DT_ANDROID_RELRSZ value (0xff) or DT_ANDROID_RELRENT value (0x18) ## Show we print a warning for an invalid relocation table entry size stored in a DT_RELRENT/DT_ANDROID_RELRENT entry. # RUN: yaml2obj --docnum=2 -DRELTYPE=RELR -DTAG1=DT_RELRSZ -DTAG2=DT_RELRENT -DTAG2VAL=0xFF %s -o %t8 @@ -149,8 +149,8 @@ ProgramHeaders: # RUN: llvm-readobj --dyn-relocations %t9 2>&1 | FileCheck %s -DFILE=%t9 --check-prefix=INVALID-DT-ANDROID-RELRENT # RUN: llvm-readelf --dyn-relocations %t9 2>&1 | FileCheck %s -DFILE=%t9 --check-prefix=INVALID-DT-ANDROID-RELRENT -## INVALID-DT-RELRENT: invalid DT_RELRSZ value (0x18) or DT_RELRENT value (0xff) -## INVALID-DT-ANDROID-RELRENT: invalid DT_ANDROID_RELRSZ value (0x18) or DT_ANDROID_RELRENT value (0xff) +# INVALID-DT-RELRENT: invalid DT_RELRSZ value (0x18) or DT_RELRENT value (0xff) +# INVALID-DT-ANDROID-RELRENT: invalid DT_ANDROID_RELRSZ value (0x18) or DT_ANDROID_RELRENT value (0xff) ## Show we print a warning for an invalid value of DT_PLTRELSZ, which describes the total size ## of the relocation entries associated with the procedure linkage table. @@ -459,3 +459,43 @@ ProgramHeaders: - Type: PT_DYNAMIC FirstSec: .dynamic LastSec: .dynamic + +## Show we print a warning for an invalid relocation table size stored in a DT_AARCH64_AUTH_RELRSZ entry. +# RUN: yaml2obj --docnum=8 -DRELTYPE=RELR -DTAG1=DT_AARCH64_AUTH_RELRSZ -DTAG1VAL=0xFF -DTAG2=DT_AARCH64_AUTH_RELRENT %s -o %t14 +# RUN: llvm-readobj --dyn-relocations %t14 2>&1 | FileCheck %s -DFILE=%t14 --check-prefix=INVALID-DT-AARCH64-AUTH-RELRSZ +# RUN: llvm-readelf --dyn-relocations %t14 2>&1 | FileCheck %s -DFILE=%t14 --check-prefix=INVALID-DT-AARCH64-AUTH-RELRSZ + +# INVALID-DT-AARCH64-AUTH-RELRSZ: warning: '[[FILE]]': invalid DT_AARCH64_AUTH_RELRSZ value (0xff) or DT_AARCH64_AUTH_RELRENT value (0x18) + +## Show we print a warning for an invalid relocation table entry size stored in a DT_AARCH64_AUTH_RELRENT entry. +# RUN: yaml2obj --docnum=8 -DRELTYPE=RELR -DTAG1=DT_AARCH64_AUTH_RELRSZ -DTAG2=DT_AARCH64_AUTH_RELRENT -DTAG2VAL=0xFF %s -o %t15 +# RUN: llvm-readobj --dyn-relocations %t15 2>&1 | FileCheck %s -DFILE=%t15 --check-prefix=INVALID-DT-AARCH64-AUTH-RELRENT +# RUN: llvm-readelf --dyn-relocations %t15 2>&1 | FileCheck %s -DFILE=%t15 --check-prefix=INVALID-DT-AARCH64-AUTH-RELRENT + +# INVALID-DT-AARCH64-AUTH-RELRENT: invalid DT_AARCH64_AUTH_RELRSZ value (0x18) or DT_AARCH64_AUTH_RELRENT value (0xff) + +--- !ELF +FileHeader: + Class: ELFCLASS64 + Data: ELFDATA2LSB + Type: ET_DYN + Machine: EM_AARCH64 +Sections: + - Name: .relx.dyn + Type: SHT_[[RELTYPE]] + - Name: .dynamic + Type: SHT_DYNAMIC + Entries: + - Tag: DT_[[RELTYPE]] + Value: 0x0 + - Tag: [[TAG1]] + Value: [[TAG1VAL=0x18]] + - Tag: [[TAG2]] + Value: [[TAG2VAL=0x18]] + - Tag: DT_NULL + Value: 0x0 +DynamicSymbols: [] +ProgramHeaders: + - Type: PT_LOAD + FirstSec: .relx.dyn + LastSec: .dynamic diff --git a/llvm/test/tools/llvm-readobj/ELF/dynamic-tags-machine-specific.test b/llvm/test/tools/llvm-readobj/ELF/dynamic-tags-machine-specific.test index c32ea33b9b3c..e7bd9cf9b48b 100644 --- a/llvm/test/tools/llvm-readobj/ELF/dynamic-tags-machine-specific.test +++ b/llvm/test/tools/llvm-readobj/ELF/dynamic-tags-machine-specific.test @@ -355,20 +355,26 @@ ProgramHeaders: # RUN: llvm-readobj --dynamic-table %t.aarch64 | FileCheck %s --check-prefix=LLVM-AARCH64 # RUN: llvm-readelf --dynamic-table %t.aarch64 | FileCheck %s --check-prefix=GNU-AARCH64 -# LLVM-AARCH64: DynamicSection [ (4 entries) +# LLVM-AARCH64: DynamicSection [ (7 entries) # LLVM-AARCH64-NEXT: Tag Type Name/Value -# LLVM-AARCH64-NEXT: 0x0000000070000001 AARCH64_BTI_PLT 1 -# LLVM-AARCH64-NEXT: 0x0000000070000003 AARCH64_PAC_PLT 2 -# LLVM-AARCH64-NEXT: 0x0000000070000005 AARCH64_VARIANT_PCS 3 -# LLVM-AARCH64-NEXT: 0x0000000000000000 NULL 0x0 +# LLVM-AARCH64-NEXT: 0x0000000070000001 AARCH64_BTI_PLT 1 +# LLVM-AARCH64-NEXT: 0x0000000070000003 AARCH64_PAC_PLT 2 +# LLVM-AARCH64-NEXT: 0x0000000070000005 AARCH64_VARIANT_PCS 3 +# LLVM-AARCH64-NEXT: 0x0000000070000012 AARCH64_AUTH_RELR 0x4 +# LLVM-AARCH64-NEXT: 0x0000000070000011 AARCH64_AUTH_RELRSZ 5 +# LLVM-AARCH64-NEXT: 0x0000000070000013 AARCH64_AUTH_RELRENT 6 +# LLVM-AARCH64-NEXT: 0x0000000000000000 NULL 0x0 # LLVM-AARCH64-NEXT:] -# GNU-AARCH64: Dynamic section at offset {{.*}} contains 4 entries: +# GNU-AARCH64: Dynamic section at offset {{.*}} contains 7 entries: # GNU-AARCH64-NEXT: Tag Type Name/Value -# GNU-AARCH64-NEXT: 0x0000000070000001 (AARCH64_BTI_PLT) 1 -# GNU-AARCH64-NEXT: 0x0000000070000003 (AARCH64_PAC_PLT) 2 -# GNU-AARCH64-NEXT: 0x0000000070000005 (AARCH64_VARIANT_PCS) 3 -# GNU-AARCH64-NEXT: 0x0000000000000000 (NULL) 0x0 +# GNU-AARCH64-NEXT: 0x0000000070000001 (AARCH64_BTI_PLT) 1 +# GNU-AARCH64-NEXT: 0x0000000070000003 (AARCH64_PAC_PLT) 2 +# GNU-AARCH64-NEXT: 0x0000000070000005 (AARCH64_VARIANT_PCS) 3 +# GNU-AARCH64-NEXT: 0x0000000070000012 (AARCH64_AUTH_RELR) 0x4 +# GNU-AARCH64-NEXT: 0x0000000070000011 (AARCH64_AUTH_RELRSZ) 5 +# GNU-AARCH64-NEXT: 0x0000000070000013 (AARCH64_AUTH_RELRENT) 6 +# GNU-AARCH64-NEXT: 0x0000000000000000 (NULL) 0x0 --- !ELF FileHeader: @@ -386,6 +392,12 @@ Sections: Value: 2 - Tag: DT_AARCH64_VARIANT_PCS Value: 3 + - Tag: DT_AARCH64_AUTH_RELR + Value: 4 + - Tag: DT_AARCH64_AUTH_RELRSZ + Value: 5 + - Tag: DT_AARCH64_AUTH_RELRENT + Value: 6 - Tag: DT_NULL Value: 0 ProgramHeaders: diff --git a/llvm/test/tools/llvm-readobj/ELF/machine-specific-section-types.test b/llvm/test/tools/llvm-readobj/ELF/machine-specific-section-types.test index 99fafe35fa44..f9524383e80b 100644 --- a/llvm/test/tools/llvm-readobj/ELF/machine-specific-section-types.test +++ b/llvm/test/tools/llvm-readobj/ELF/machine-specific-section-types.test @@ -53,11 +53,14 @@ # MIPS-GNU: abiflags MIPS_ABIFLAGS # MIPS-GNU: dwarf MIPS_DWARF +# AARCH64-LLVM: Name: aarch64_auth_relr +# AARCH64-LLVM: Type: SHT_AARCH64_AUTH_RELR # AARCH64-LLVM: Name: .memtag.globals.dynamic # AARCH64-LLVM: Type: SHT_AARCH64_MEMTAG_GLOBALS_DYNAMIC # AARCH64-LLVM: Name: .memtag.globals.static # AARCH64-LLVM: Type: SHT_AARCH64_MEMTAG_GLOBALS_STATIC +# AARCH64-GNU: aarch64_auth_relr AARCH64_AUTH_RELR # AARCH64-GNU: .memtag.globals.dynamic AARCH64_MEMTAG_GLOBALS_DYNAMIC # AARCH64-GNU: .memtag.globals.static AARCH64_MEMTAG_GLOBALS_STATIC @@ -113,6 +116,8 @@ FileHeader: Type: ET_REL Machine: EM_AARCH64 Sections: + - Name: aarch64_auth_relr + Type: SHT_AARCH64_AUTH_RELR - Name: .memtag.globals.dynamic Type: SHT_AARCH64_MEMTAG_GLOBALS_DYNAMIC - Name: .memtag.globals.static diff --git a/llvm/test/tools/llvm-readobj/ELF/needed-libs.test b/llvm/test/tools/llvm-readobj/ELF/needed-libs.test index d37ab6238e13..d2d0bdad26ed 100644 --- a/llvm/test/tools/llvm-readobj/ELF/needed-libs.test +++ b/llvm/test/tools/llvm-readobj/ELF/needed-libs.test @@ -5,6 +5,8 @@ # RUN: | FileCheck %s -DFILE=%t1 --implicit-check-not=warning: --strict-whitespace --check-prefix=NEEDED-LIBS # RUN: llvm-readelf --needed-libs %t1 2>&1 \ # RUN: | FileCheck %s -DFILE=%t1 --implicit-check-not=warning: --strict-whitespace --check-prefix=NEEDED-LIBS +# RUN: llvm-readobj --elf-output-style=JSON --needed-libs --pretty-print %t1 2>&1 \ +# RUN: | FileCheck %s -DFILE=%t1 --implicit-check-not=warning: --match-full-lines --strict-whitespace --check-prefix=JSON ## Check that library names are sorted when printed. ## Document that we also sort error entries. @@ -19,6 +21,17 @@ # NEEDED-LIBS-NEXT:{{^}} ccc{{$}} # NEEDED-LIBS-NEXT:{{^}}]{{$}} +# JSON: "FileSummary": { +# JSON: "NeededLibraries": [{{.*}}warning: '[[FILE]]': string table at offset 0x78: unable to read the string at 0x9999a11: it goes past the end of the table (0x85) +# JSON-NEXT:{{.*}}warning: '[[FILE]]': string table at offset 0x78: unable to read the string at 0x1111189: it goes past the end of the table (0x85) +# JSON-EMPTY: +# JSON-NEXT: "", +# JSON-NEXT: "", +# JSON-NEXT: "aaa", +# JSON-NEXT: "bbb", +# JSON-NEXT: "ccc" +# JSON-NEXT: ] + --- !ELF FileHeader: Class: ELFCLASS64 diff --git a/llvm/test/tools/llvm-readobj/ELF/relr-relocs.test b/llvm/test/tools/llvm-readobj/ELF/relr-relocs.test index 3bb54b1adc1f..91b148ebb6e3 100644 --- a/llvm/test/tools/llvm-readobj/ELF/relr-relocs.test +++ b/llvm/test/tools/llvm-readobj/ELF/relr-relocs.test @@ -158,7 +158,7 @@ Sections: Link: [[LINK=]] ## Check we report a warning when we are unable to dump relocations -## for a SHT_RELR/SHT_ANDROID_RELR section. +## for a SHT_RELR/SHT_ANDROID_RELR/SHT_AARCH64_AUTH_RELR section. ## Case A: check the case when relocations can't be read from an SHT_RELR section. # RUN: yaml2obj --docnum=2 -DENTSIZE=1 %s -o %t2.broken @@ -186,7 +186,20 @@ Sections: # RUN: llvm-readelf --relocations %t2.broken.android 2>&1 | \ # RUN: FileCheck -DFILE=%t2.broken.android --check-prefix=BROKEN-GNU %s -DSECNAME=SHT_ANDROID_RELR -## Check the behavior when the sh_link field of the SHT_RELR/SHT_ANDROID_RELR section +## Case C: check the case when we have an SHT_AARCH64_AUTH_RELR section in non-AArch64 ELF. +## SHT_AARCH64_AUTH_RELR = 0x70000004. +# RUN: yaml2obj --docnum=2 -DENTSIZE=1 -DSHTYPE=0x70000004 %s -o %t2.broken.aarch64auth +# RUN: llvm-readobj --relocations %t2.broken.aarch64auth 2>&1 | \ +# RUN: FileCheck -DFILE=%t2.broken.aarch64auth --check-prefix=WRONGARCH-LLVM-AARCH64-AUTH %s -DSECNAME=SHT_AARCH64_AUTH_RELR +# RUN: llvm-readelf --relocations %t2.broken.aarch64auth 2>&1 | \ +# RUN: FileCheck -DFILE=%t2.broken.aarch64auth --check-prefix=WRONGARCH-GNU-AARCH64-AUTH %s -DSECNAME=SHT_AARCH64_AUTH_RELR + +# WRONGARCH-LLVM-AARCH64-AUTH: Relocations [ +# WRONGARCH-LLVM-AARCH64-AUTH-NEXT: ] + +# WRONGARCH-GNU-AARCH64-AUTH-NOT: Relocation section + +## Check the behavior when the sh_link field of the SHT_RELR/SHT_ANDROID_RELR/SHT_AARCH64_AUTH_RELR section ## is set to an arbitrary value. Normally, it is set to 0, because such sections contains ## only relative relocations and do not have an associated symbol table, like other ## relocation sections. diff --git a/llvm/test/tools/obj2yaml/XCOFF/aix.yaml b/llvm/test/tools/obj2yaml/XCOFF/aix.yaml index cd1e88dec11d..fbd5fa0629d1 100644 --- a/llvm/test/tools/obj2yaml/XCOFF/aix.yaml +++ b/llvm/test/tools/obj2yaml/XCOFF/aix.yaml @@ -52,12 +52,30 @@ # CHECK32-NEXT: Type: 0x0 # CHECK32-NEXT: StorageClass: C_EXT # CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_CSECT +# CHECK32-NEXT: ParameterHashIndex: 0 +# CHECK32-NEXT: TypeChkSectNum: 0 +# CHECK32-NEXT: SymbolAlignmentAndType: 0 +# CHECK32-NEXT: StorageMappingClass: XMC_PR +# CHECK32-NEXT: SectionOrLength: 0 +# CHECK32-NEXT: StabInfoIndex: 0 +# CHECK32-NEXT: StabSectNum: 0 # CHECK32-NEXT: - Name: .data # CHECK32-NEXT: Value: 0x70 # CHECK32-NEXT: Section: .data # CHECK32-NEXT: Type: 0x0 # CHECK32-NEXT: StorageClass: C_HIDEXT # CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_CSECT +# CHECK32-NEXT: ParameterHashIndex: 0 +# CHECK32-NEXT: TypeChkSectNum: 0 +# CHECK32-NEXT: SymbolAlignmentAndType: 0 +# CHECK32-NEXT: StorageMappingClass: XMC_PR +# CHECK32-NEXT: SectionOrLength: 0 +# CHECK32-NEXT: StabInfoIndex: 0 +# CHECK32-NEXT: StabSectNum: 0 # CHECK64: --- !XCOFF # CHECK64-NEXT: FileHeader: @@ -106,12 +124,28 @@ # CHECK64-NEXT: Type: 0x0 # CHECK64-NEXT: StorageClass: C_EXT # CHECK64-NEXT: NumberOfAuxEntries: 1 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_CSECT +# CHECK64-NEXT: ParameterHashIndex: 0 +# CHECK64-NEXT: TypeChkSectNum: 0 +# CHECK64-NEXT: SymbolAlignmentAndType: 0 +# CHECK64-NEXT: StorageMappingClass: XMC_PR +# CHECK64-NEXT: SectionOrLengthLo: 0 +# CHECK64-NEXT: SectionOrLengthHi: 0 # CHECK64-NEXT: - Name: .data # CHECK64-NEXT: Value: 0x70 # CHECK64-NEXT: Section: .data # CHECK64-NEXT: Type: 0x0 # CHECK64-NEXT: StorageClass: C_HIDEXT # CHECK64-NEXT: NumberOfAuxEntries: 1 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_CSECT +# CHECK64-NEXT: ParameterHashIndex: 0 +# CHECK64-NEXT: TypeChkSectNum: 0 +# CHECK64-NEXT: SymbolAlignmentAndType: 0 +# CHECK64-NEXT: StorageMappingClass: XMC_PR +# CHECK64-NEXT: SectionOrLengthLo: 0 +# CHECK64-NEXT: SectionOrLengthHi: 0 --- !XCOFF FileHeader: @@ -140,9 +174,13 @@ Symbols: Type: 0x0 StorageClass: C_EXT NumberOfAuxEntries: 1 + AuxEntries: + - Type: AUX_CSECT - Name: .data Value: 0x70 Section: .data Type: 0x0 StorageClass: C_HIDEXT NumberOfAuxEntries: 1 + AuxEntries: + - Type: AUX_CSECT diff --git a/llvm/test/tools/obj2yaml/XCOFF/aux-symbols.yaml b/llvm/test/tools/obj2yaml/XCOFF/aux-symbols.yaml new file mode 100644 index 000000000000..7f93b8dae0ca --- /dev/null +++ b/llvm/test/tools/obj2yaml/XCOFF/aux-symbols.yaml @@ -0,0 +1,322 @@ +## Check that obj2yaml can parse auxiliary symbols for XCOFF object file correctly. + +## 32-bit +# RUN: yaml2obj %s --docnum=1 -o %t32 +# RUN: obj2yaml %t32 | FileCheck %s --check-prefix=CHECK32 + +# CHECK32: --- !XCOFF +# CHECK32-NEXT: FileHeader: +# CHECK32-NEXT: MagicNumber: 0x1DF +# CHECK32-NEXT: NumberOfSections: 0 +# CHECK32-NEXT: CreationTime: 0 +# CHECK32-NEXT: OffsetToSymbolTable: 0x14 +# CHECK32-NEXT: EntriesInSymbolTable: 13 +# CHECK32-NEXT: AuxiliaryHeaderSize: 0 +# CHECK32-NEXT: Flags: 0x0 +# CHECK32-NEXT: Symbols: +# CHECK32-NEXT: - Name: '' +# CHECK32-NEXT: Value: 0x0 +# CHECK32-NEXT: Section: N_UNDEF +# CHECK32-NEXT: Type: 0x0 +# CHECK32-NEXT: StorageClass: C_FILE +# CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_FILE +# CHECK32-NEXT: FileNameOrString: FileName +# CHECK32-NEXT: FileStringType: XFT_CD +# CHECK32-NEXT: - Name: '' +# CHECK32-NEXT: Value: 0x0 +# CHECK32-NEXT: Section: N_UNDEF +# CHECK32-NEXT: Type: 0x0 +# CHECK32-NEXT: StorageClass: C_HIDEXT +# CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_CSECT +# CHECK32-NEXT: ParameterHashIndex: 1 +# CHECK32-NEXT: TypeChkSectNum: 2 +# CHECK32-NEXT: SymbolAlignmentAndType: 41 +# CHECK32-NEXT: StorageMappingClass: XMC_PR +# CHECK32-NEXT: SectionOrLength: 3 +# CHECK32-NEXT: StabInfoIndex: 4 +# CHECK32-NEXT: StabSectNum: 5 +# CHECK32-NEXT: - Name: '' +# CHECK32-NEXT: Value: 0x0 +# CHECK32-NEXT: Section: N_UNDEF +# CHECK32-NEXT: Type: 0x0 +# CHECK32-NEXT: StorageClass: C_EXT +# CHECK32-NEXT: NumberOfAuxEntries: 2 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_FCN +# CHECK32-NEXT: OffsetToExceptionTbl: 1 +# CHECK32-NEXT: SizeOfFunction: 2 +# CHECK32-NEXT: SymIdxOfNextBeyond: 3 +# CHECK32-NEXT: PtrToLineNum: 4 +# CHECK32-NEXT: - Type: AUX_CSECT +# CHECK32-NEXT: ParameterHashIndex: 1 +# CHECK32-NEXT: TypeChkSectNum: 2 +# CHECK32-NEXT: SymbolAlignmentAndType: 17 +# CHECK32-NEXT: StorageMappingClass: XMC_PR +# CHECK32-NEXT: SectionOrLength: 4 +# CHECK32-NEXT: StabInfoIndex: 5 +# CHECK32-NEXT: StabSectNum: 6 +# CHECK32-NEXT: - Name: '' +# CHECK32-NEXT: Value: 0x0 +# CHECK32-NEXT: Section: N_UNDEF +# CHECK32-NEXT: Type: 0x0 +# CHECK32-NEXT: StorageClass: C_DWARF +# CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_SECT +# CHECK32-NEXT: LengthOfSectionPortion: 44 +# CHECK32-NEXT: NumberOfRelocEnt: 1 +# CHECK32-NEXT: - Name: '' +# CHECK32-NEXT: Value: 0x0 +# CHECK32-NEXT: Section: N_UNDEF +# CHECK32-NEXT: Type: 0x0 +# CHECK32-NEXT: StorageClass: C_STAT +# CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_STAT +# CHECK32-NEXT: SectionLength: 1 +# CHECK32-NEXT: NumberOfRelocEnt: 2 +# CHECK32-NEXT: NumberOfLineNum: 3 +# CHECK32-NEXT: - Name: '' +# CHECK32-NEXT: Value: 0x0 +# CHECK32-NEXT: Section: N_UNDEF +# CHECK32-NEXT: Type: 0x0 +# CHECK32-NEXT: StorageClass: C_BLOCK +# CHECK32-NEXT: NumberOfAuxEntries: 1 +# CHECK32-NEXT: AuxEntries: +# CHECK32-NEXT: - Type: AUX_SYM +# CHECK32-NEXT: LineNumHi: 1 +# CHECK32-NEXT: LineNumLo: 2 + +--- !XCOFF +FileHeader: + MagicNumber: 0x01DF +Symbols: + - StorageClass: C_FILE + AuxEntries: + - Type: AUX_FILE + FileNameOrString: FileName + FileStringType: XFT_CD + - StorageClass: C_HIDEXT + AuxEntries: + - Type: AUX_CSECT + ParameterHashIndex: 1 + TypeChkSectNum: 2 + SymbolAlignmentAndType: 41 + SectionOrLength: 3 + StabInfoIndex: 4 + StabSectNum: 5 + - StorageClass: C_EXT + AuxEntries: + - Type: AUX_FCN + OffsetToExceptionTbl: 1 + SizeOfFunction: 2 + SymIdxOfNextBeyond: 3 + PtrToLineNum: 4 + - Type: AUX_CSECT + ParameterHashIndex: 1 + TypeChkSectNum: 2 + SymbolAlignmentAndType: 17 + SectionOrLength: 4 + StabInfoIndex: 5 + StabSectNum: 6 + - StorageClass: C_DWARF + AuxEntries: + - Type: AUX_SECT + LengthOfSectionPortion: 44 + NumberOfRelocEnt: 1 + - StorageClass: C_STAT + AuxEntries: + - Type: AUX_STAT + SectionLength: 1 + NumberOfRelocEnt: 2 + NumberOfLineNum: 3 + - StorageClass: C_BLOCK + AuxEntries: + - Type: AUX_SYM + LineNumHi: 1 + LineNumLo: 2 + +## 64-bit +# RUN: yaml2obj %s --docnum=2 -o %t64 +# RUN: obj2yaml %t64 | FileCheck %s --check-prefix=CHECK64 + +# CHECK64: --- !XCOFF +# CHECK64-NEXT: FileHeader: +# CHECK64-NEXT: MagicNumber: 0x1F7 +# CHECK64-NEXT: NumberOfSections: 0 +# CHECK64-NEXT: CreationTime: 0 +# CHECK64-NEXT: OffsetToSymbolTable: 0x18 +# CHECK64-NEXT: EntriesInSymbolTable: 12 +# CHECK64-NEXT: AuxiliaryHeaderSize: 0 +# CHECK64-NEXT: Flags: 0x0 +# CHECK64-NEXT: Symbols: +# CHECK64-NEXT: - Name: '' +# CHECK64-NEXT: Value: 0x0 +# CHECK64-NEXT: Section: N_UNDEF +# CHECK64-NEXT: Type: 0x0 +# CHECK64-NEXT: StorageClass: C_FILE +# CHECK64-NEXT: NumberOfAuxEntries: 1 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_FILE +# CHECK64-NEXT: FileNameOrString: FileName +# CHECK64-NEXT: FileStringType: XFT_CD +# CHECK64-NEXT: - Name: '' +# CHECK64-NEXT: Value: 0x0 +# CHECK64-NEXT: Section: N_UNDEF +# CHECK64-NEXT: Type: 0x0 +# CHECK64-NEXT: StorageClass: C_HIDEXT +# CHECK64-NEXT: NumberOfAuxEntries: 1 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_CSECT +# CHECK64-NEXT: ParameterHashIndex: 1 +# CHECK64-NEXT: TypeChkSectNum: 2 +# CHECK64-NEXT: SymbolAlignmentAndType: 41 +# CHECK64-NEXT: StorageMappingClass: XMC_PR +# CHECK64-NEXT: SectionOrLengthLo: 3 +# CHECK64-NEXT: SectionOrLengthHi: 4 +# CHECK64-NEXT: - Name: '' +# CHECK64-NEXT: Value: 0x0 +# CHECK64-NEXT: Section: N_UNDEF +# CHECK64-NEXT: Type: 0x0 +# CHECK64-NEXT: StorageClass: C_EXT +# CHECK64-NEXT: NumberOfAuxEntries: 3 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_FCN +# CHECK64-NEXT: SizeOfFunction: 3 +# CHECK64-NEXT: SymIdxOfNextBeyond: 2 +# CHECK64-NEXT: PtrToLineNum: 1 +# CHECK64-NEXT: - Type: AUX_EXCEPT +# CHECK64-NEXT: OffsetToExceptionTbl: 1 +# CHECK64-NEXT: SizeOfFunction: 2 +# CHECK64-NEXT: SymIdxOfNextBeyond: 3 +# CHECK64-NEXT: - Type: AUX_CSECT +# CHECK64-NEXT: ParameterHashIndex: 1 +# CHECK64-NEXT: TypeChkSectNum: 2 +# CHECK64-NEXT: SymbolAlignmentAndType: 17 +# CHECK64-NEXT: StorageMappingClass: XMC_PR +# CHECK64-NEXT: SectionOrLengthLo: 3 +# CHECK64-NEXT: SectionOrLengthHi: 4 +# CHECK64-NEXT: - Name: '' +# CHECK64-NEXT: Value: 0x0 +# CHECK64-NEXT: Section: N_UNDEF +# CHECK64-NEXT: Type: 0x0 +# CHECK64-NEXT: StorageClass: C_DWARF +# CHECK64-NEXT: NumberOfAuxEntries: 1 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_SECT +# CHECK64-NEXT: LengthOfSectionPortion: 44 +# CHECK64-NEXT: NumberOfRelocEnt: 1 +# CHECK64-NEXT: - Name: '' +# CHECK64-NEXT: Value: 0x0 +# CHECK64-NEXT: Section: N_UNDEF +# CHECK64-NEXT: Type: 0x0 +# CHECK64-NEXT: StorageClass: C_BLOCK +# CHECK64-NEXT: NumberOfAuxEntries: 1 +# CHECK64-NEXT: AuxEntries: +# CHECK64-NEXT: - Type: AUX_SYM +# CHECK64-NEXT: LineNum: 1 + +--- !XCOFF +FileHeader: + MagicNumber: 0x1F7 +Symbols: + - StorageClass: C_FILE + AuxEntries: + - Type: AUX_FILE + FileNameOrString: FileName + FileStringType: XFT_CD + - StorageClass: C_HIDEXT + AuxEntries: + - Type: AUX_CSECT + ParameterHashIndex: 1 + TypeChkSectNum: 2 + SymbolAlignmentAndType: 41 + SectionOrLengthLo: 3 + SectionOrLengthHi: 4 + - StorageClass: C_EXT + AuxEntries: + - Type: AUX_FCN + SizeOfFunction: 3 + SymIdxOfNextBeyond: 2 + PtrToLineNum: 1 + - Type: AUX_EXCEPT + OffsetToExceptionTbl: 1 + SizeOfFunction: 2 + SymIdxOfNextBeyond: 3 + - Type: AUX_CSECT + ParameterHashIndex: 1 + TypeChkSectNum: 2 + SymbolAlignmentAndType: 17 + SectionOrLengthLo: 3 + SectionOrLengthHi: 4 + - StorageClass: C_DWARF + AuxEntries: + - Type: AUX_SECT + LengthOfSectionPortion: 44 + NumberOfRelocEnt: 1 + - StorageClass: C_BLOCK + AuxEntries: + - Type: AUX_SYM + LineNum: 1 + +## Check that error messages are reported appropriately. + +# RUN: yaml2obj %s --docnum=3 -o - | not obj2yaml 2>&1 | FileCheck %s --check-prefix=ERROR1 +# ERROR1: failed to parse symbol "SYM_STAT" with index of 1: expected 1 aux symbol for C_STAT, while got 2 + +--- !XCOFF +FileHeader: + MagicNumber: 0x1DF +Symbols: + - Name: SYM + - Name: SYM_STAT + StorageClass: C_STAT + NumberOfAuxEntries: 2 + AuxEntries: + - Type: AUX_STAT + - Type: AUX_STAT + +# RUN: yaml2obj %s --docnum=4 -o - | not obj2yaml 2>&1 | FileCheck %s --check-prefix=ERROR2 +# ERROR2: failed to parse symbol "SYM_BLOCK" with index of 0: expected 1 aux symbol for C_BLOCK or C_FCN, while got 2 + +--- !XCOFF +FileHeader: + MagicNumber: 0x1DF +Symbols: + - Name: SYM_BLOCK + StorageClass: C_BLOCK + NumberOfAuxEntries: 2 + AuxEntries: + - Type: AUX_SYM + - Type: AUX_SYM + +# RUN: yaml2obj %s --docnum=5 -o - | not obj2yaml 2>&1 | FileCheck %s --check-prefix=ERROR3 +# ERROR3: failed to parse symbol "SYM_DWARF" with index of 0: expected 1 aux symbol for C_DWARF, while got 2 + +--- !XCOFF +FileHeader: + MagicNumber: 0x1DF +Symbols: + - Name: SYM_DWARF + StorageClass: C_DWARF + NumberOfAuxEntries: 2 + AuxEntries: + - Type: AUX_SECT + - Type: AUX_SECT + +# RUN: yaml2obj %s --docnum=6 -o - | not obj2yaml 2>&1 | FileCheck %s --check-prefix=ERROR4 +# ERROR4: failed to parse symbol "SYM_EXT" with index of 0: invalid auxiliary symbol type: 252 + +--- !XCOFF +FileHeader: + MagicNumber: 0x1F7 +Symbols: + - Name: SYM_EXT + StorageClass: C_EXT + AuxEntries: + - Type: AUX_FILE + - Type: AUX_CSECT diff --git a/llvm/tools/bugpoint-passes/CMakeLists.txt b/llvm/tools/bugpoint-passes/CMakeLists.txt index 6df49d7abd88..60fc1bde5137 100644 --- a/llvm/tools/bugpoint-passes/CMakeLists.txt +++ b/llvm/tools/bugpoint-passes/CMakeLists.txt @@ -10,7 +10,7 @@ if( NOT LLVM_REQUIRES_RTTI ) endif() endif() -if(WIN32 OR CYGWIN) +if(WIN32 OR CYGWIN OR ZOS) set(LLVM_LINK_COMPONENTS Core Support) endif() diff --git a/llvm/tools/dsymutil/DebugMap.cpp b/llvm/tools/dsymutil/DebugMap.cpp index dcdecdfe8210..8724b70422f3 100644 --- a/llvm/tools/dsymutil/DebugMap.cpp +++ b/llvm/tools/dsymutil/DebugMap.cpp @@ -287,7 +287,7 @@ MappingTraits::YamlDMO::denormalize(IO &IO) { } uint8_t Type = MachO::N_OSO; - if (Path.endswith(".dylib")) { + if (Path.ends_with(".dylib")) { // FIXME: find a more resilient way Type = MachO::N_LIB; } diff --git a/llvm/tools/llvm-c-test/echo.cpp b/llvm/tools/llvm-c-test/echo.cpp index e2617583ff9b..bfc14e85a12b 100644 --- a/llvm/tools/llvm-c-test/echo.cpp +++ b/llvm/tools/llvm-c-test/echo.cpp @@ -548,16 +548,26 @@ struct FunCloner { break; case LLVMInvoke: { SmallVector Args; - int ArgCount = LLVMGetNumArgOperands(Src); - for (int i = 0; i < ArgCount; i++) + SmallVector Bundles; + unsigned ArgCount = LLVMGetNumArgOperands(Src); + for (unsigned i = 0; i < ArgCount; ++i) Args.push_back(CloneValue(LLVMGetOperand(Src, i))); + unsigned BundleCount = LLVMGetNumOperandBundles(Src); + for (unsigned i = 0; i < BundleCount; ++i) { + auto Bundle = LLVMGetOperandBundleAtIndex(Src, i); + Bundles.push_back(CloneOB(Bundle)); + LLVMDisposeOperandBundle(Bundle); + } LLVMTypeRef FnTy = CloneType(LLVMGetCalledFunctionType(Src)); LLVMValueRef Fn = CloneValue(LLVMGetCalledValue(Src)); LLVMBasicBlockRef Then = DeclareBB(LLVMGetNormalDest(Src)); LLVMBasicBlockRef Unwind = DeclareBB(LLVMGetUnwindDest(Src)); - Dst = LLVMBuildInvoke2(Builder, FnTy, Fn, Args.data(), ArgCount, - Then, Unwind, Name); + Dst = LLVMBuildInvokeWithOperandBundles( + Builder, FnTy, Fn, Args.data(), ArgCount, Then, Unwind, + Bundles.data(), Bundles.size(), Name); CloneAttrs(Src, Dst); + for (auto Bundle : Bundles) + LLVMDisposeOperandBundle(Bundle); break; } case LLVMUnreachable: @@ -764,14 +774,25 @@ struct FunCloner { } case LLVMCall: { SmallVector Args; - int ArgCount = LLVMGetNumArgOperands(Src); - for (int i = 0; i < ArgCount; i++) + SmallVector Bundles; + unsigned ArgCount = LLVMGetNumArgOperands(Src); + for (unsigned i = 0; i < ArgCount; ++i) Args.push_back(CloneValue(LLVMGetOperand(Src, i))); + unsigned BundleCount = LLVMGetNumOperandBundles(Src); + for (unsigned i = 0; i < BundleCount; ++i) { + auto Bundle = LLVMGetOperandBundleAtIndex(Src, i); + Bundles.push_back(CloneOB(Bundle)); + LLVMDisposeOperandBundle(Bundle); + } LLVMTypeRef FnTy = CloneType(LLVMGetCalledFunctionType(Src)); LLVMValueRef Fn = CloneValue(LLVMGetCalledValue(Src)); - Dst = LLVMBuildCall2(Builder, FnTy, Fn, Args.data(), ArgCount, Name); + Dst = LLVMBuildCallWithOperandBundles(Builder, FnTy, Fn, Args.data(), + ArgCount, Bundles.data(), + Bundles.size(), Name); LLVMSetTailCallKind(Dst, LLVMGetTailCallKind(Src)); CloneAttrs(Src, Dst); + for (auto Bundle : Bundles) + LLVMDisposeOperandBundle(Bundle); break; } case LLVMResume: { @@ -935,6 +956,17 @@ struct FunCloner { return VMap[Src] = Dst; } + LLVMOperandBundleRef CloneOB(LLVMOperandBundleRef Src) { + size_t TagLen; + const char *Tag = LLVMGetOperandBundleTag(Src, &TagLen); + + SmallVector Args; + for (unsigned i = 0, n = LLVMGetNumOperandBundleArgs(Src); i != n; ++i) + Args.push_back(CloneValue(LLVMGetOperandBundleArgAtIndex(Src, i))); + + return LLVMCreateOperandBundle(Tag, TagLen, Args.data(), Args.size()); + } + LLVMBasicBlockRef DeclareBB(LLVMBasicBlockRef Src) { // Check if this is something we already computed. { diff --git a/llvm/tools/llvm-cov/CodeCoverage.cpp b/llvm/tools/llvm-cov/CodeCoverage.cpp index b5d763d8643c..c1e3e2c4d1e2 100644 --- a/llvm/tools/llvm-cov/CodeCoverage.cpp +++ b/llvm/tools/llvm-cov/CodeCoverage.cpp @@ -502,7 +502,7 @@ void CodeCoverageTool::remapPathNames(const CoverageMapping &Coverage) { SmallString<128> NativeFilename; sys::path::native(Filename, NativeFilename); sys::path::remove_dots(NativeFilename, true); - if (NativeFilename.startswith(RemapFrom)) { + if (NativeFilename.starts_with(RemapFrom)) { RemappedFilenames[Filename] = RemapTo + NativeFilename.substr(RemapFrom.size()).str(); } diff --git a/llvm/tools/llvm-cov/CoverageReport.cpp b/llvm/tools/llvm-cov/CoverageReport.cpp index 060733b2d5d6..eae2653b85c8 100644 --- a/llvm/tools/llvm-cov/CoverageReport.cpp +++ b/llvm/tools/llvm-cov/CoverageReport.cpp @@ -211,7 +211,7 @@ void CoverageReport::render(const FileCoverageSummary &File, sys::path::native(FileName); // remove_dots will remove trailing slash, so we need to check before it. - auto IsDir = FileName.endswith(sys::path::get_separator()); + auto IsDir = FileName.ends_with(sys::path::get_separator()); sys::path::remove_dots(FileName, /*remove_dot_dot=*/true); if (IsDir) FileName += sys::path::get_separator(); diff --git a/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp b/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp index 79a0494815c2..d0a2e44be252 100644 --- a/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp +++ b/llvm/tools/llvm-cov/SourceCoverageViewHTML.cpp @@ -639,7 +639,7 @@ struct CoveragePrinterHTMLDirectory::Reporter : public DirectoryCoverageReport { sys::path::native(LinkTextStr); // remove_dots will remove trailing slash, so we need to check before it. - auto IsDir = LinkTextStr.endswith(sys::path::get_separator()); + auto IsDir = LinkTextStr.ends_with(sys::path::get_separator()); sys::path::remove_dots(LinkTextStr, /*remove_dot_dot=*/true); SmallString<128> LinkTargetStr(LinkTextStr); diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h index 38111519a2c8..0d08febae20c 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkResult.h @@ -70,6 +70,9 @@ struct BenchmarkKey { // An opaque configuration, that can be used to separate several benchmarks of // the same instruction under different configurations. std::string Config; + // The address that the snippet should be loaded in at if the execution mode + // being used supports it. + intptr_t SnippetAddress = 0; }; struct BenchmarkMeasure { diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp index dd7c1a315531..6c34446e8d66 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.cpp @@ -44,7 +44,14 @@ #define GLIBC_INITS_RSEQ #endif #endif + +// Before kernel 4.17, Linux did not support MAP_FIXED_NOREPLACE, so if it is +// not available, simplfy define it as MAP_FIXED which performs the same +// function but does not guarantee existing mappings won't get clobbered. +#ifndef MAP_FIXED_NOREPLACE +#define MAP_FIXED_NOREPLACE MAP_FIXED #endif +#endif // __linux__ namespace llvm { namespace exegesis { @@ -416,9 +423,17 @@ private: #endif // GLIBC_INITS_RSEQ size_t FunctionDataCopySize = this->Function.FunctionBytes.size(); + void *MapAddress = NULL; + int MapFlags = MAP_PRIVATE | MAP_ANONYMOUS; + + if (Key.SnippetAddress != 0) { + MapAddress = reinterpret_cast(Key.SnippetAddress); + MapFlags |= MAP_FIXED_NOREPLACE; + } + char *FunctionDataCopy = - (char *)mmap(NULL, FunctionDataCopySize, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + (char *)mmap(MapAddress, FunctionDataCopySize, PROT_READ | PROT_WRITE, + MapFlags, 0, 0); if ((intptr_t)FunctionDataCopy == -1) exit(ChildProcessExitCodeE::FunctionDataMappingFailed); @@ -557,7 +572,7 @@ BenchmarkRunner::createFunctionExecutor( llvm_unreachable("ExecutionMode is outside expected range"); } -Expected BenchmarkRunner::runConfiguration( +std::pair BenchmarkRunner::runConfiguration( RunnableConfiguration &&RC, const std::optional &DumpFile) const { Benchmark &InstrBenchmark = RC.InstrBenchmark; @@ -568,8 +583,7 @@ Expected BenchmarkRunner::runConfiguration( auto ObjectFilePath = writeObjectFile(ObjectFile.getBinary()->getData(), *DumpFile); if (Error E = ObjectFilePath.takeError()) { - InstrBenchmark.Error = toString(std::move(E)); - return std::move(InstrBenchmark); + return {std::move(E), std::move(InstrBenchmark)}; } outs() << "Check generated assembly with: /usr/bin/objdump -d " << *ObjectFilePath << "\n"; @@ -577,20 +591,17 @@ Expected BenchmarkRunner::runConfiguration( if (BenchmarkPhaseSelector < BenchmarkPhaseSelectorE::Measure) { InstrBenchmark.Error = "actual measurements skipped."; - return std::move(InstrBenchmark); + return {Error::success(), std::move(InstrBenchmark)}; } Expected> Executor = createFunctionExecutor(std::move(ObjectFile), RC.InstrBenchmark.Key); if (!Executor) - return Executor.takeError(); + return {Executor.takeError(), std::move(InstrBenchmark)}; auto NewMeasurements = runMeasurements(**Executor); if (Error E = NewMeasurements.takeError()) { - if (!E.isA()) - return std::move(E); - InstrBenchmark.Error = toString(std::move(E)); - return std::move(InstrBenchmark); + return {std::move(E), std::move(InstrBenchmark)}; } assert(InstrBenchmark.NumRepetitions > 0 && "invalid NumRepetitions"); for (BenchmarkMeasure &BM : *NewMeasurements) { @@ -603,7 +614,7 @@ Expected BenchmarkRunner::runConfiguration( } InstrBenchmark.Measurements = std::move(*NewMeasurements); - return std::move(InstrBenchmark); + return {Error::success(), std::move(InstrBenchmark)}; } Expected diff --git a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h index 24f208628940..2c48d07e37ca 100644 --- a/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h +++ b/llvm/tools/llvm-exegesis/lib/BenchmarkRunner.h @@ -65,7 +65,7 @@ public: unsigned NumRepetitions, unsigned LoopUnrollFactor, const SnippetRepetitor &Repetitor) const; - Expected + std::pair runConfiguration(RunnableConfiguration &&RC, const std::optional &DumpFile) const; diff --git a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp index 753efed13816..eda450579a58 100644 --- a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp +++ b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.cpp @@ -21,11 +21,13 @@ namespace exegesis { LatencyBenchmarkRunner::LatencyBenchmarkRunner( const LLVMState &State, Benchmark::ModeE Mode, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, - Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode) + Benchmark::ResultAggregationModeE ResultAgg, ExecutionModeE ExecutionMode, + unsigned BenchmarkRepeatCount) : BenchmarkRunner(State, Mode, BenchmarkPhaseSelector, ExecutionMode) { assert((Mode == Benchmark::Latency || Mode == Benchmark::InverseThroughput) && "invalid mode"); ResultAggMode = ResultAgg; + NumMeasurements = BenchmarkRepeatCount; } LatencyBenchmarkRunner::~LatencyBenchmarkRunner() = default; @@ -68,7 +70,6 @@ Expected> LatencyBenchmarkRunner::runMeasurements( // Cycle measurements include some overhead from the kernel. Repeat the // measure several times and return the aggregated value, as specified by // ResultAggMode. - constexpr const int NumMeasurements = 30; llvm::SmallVector AccumulatedValues; double MinVariance = std::numeric_limits::infinity(); const char *CounterName = State.getPfmCounters().CycleCounter; diff --git a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h index 34b912f0abde..fc159d7d9b5e 100644 --- a/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h +++ b/llvm/tools/llvm-exegesis/lib/LatencyBenchmarkRunner.h @@ -24,7 +24,8 @@ public: LatencyBenchmarkRunner(const LLVMState &State, Benchmark::ModeE Mode, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, Benchmark::ResultAggregationModeE ResultAggMode, - ExecutionModeE ExecutionMode); + ExecutionModeE ExecutionMode, + unsigned BenchmarkRepeatCount); ~LatencyBenchmarkRunner() override; private: @@ -32,6 +33,7 @@ private: runMeasurements(const FunctionExecutor &Executor) const override; Benchmark::ResultAggregationModeE ResultAggMode; + unsigned NumMeasurements; }; } // namespace exegesis } // namespace llvm diff --git a/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp b/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp index d85a9f190655..286c606cd6fb 100644 --- a/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp +++ b/llvm/tools/llvm-exegesis/lib/SnippetFile.cpp @@ -131,6 +131,19 @@ public: Result->Key.MemoryMappings.push_back(std::move(MemMap)); return; } + if (CommentText.consume_front("SNIPPET-ADDRESS")) { + // LLVM-EXEGESIS-SNIPPET-ADDRESS

+ if (!to_integer(CommentText.trim(), Result->Key.SnippetAddress, + 16)) { + errs() << "invalid comment 'LLVM-EXEGESIS-SNIPPET-ADDRESS " + << CommentText + << "', expected
to contain a valid integer in " + "hexadecimal format"; + ++InvalidComments; + return; + } + return; + } } unsigned numInvalidComments() const { return InvalidComments; } diff --git a/llvm/tools/llvm-exegesis/lib/Target.cpp b/llvm/tools/llvm-exegesis/lib/Target.cpp index aed093548f15..23c80e5b9895 100644 --- a/llvm/tools/llvm-exegesis/lib/Target.cpp +++ b/llvm/tools/llvm-exegesis/lib/Target.cpp @@ -79,6 +79,7 @@ ExegesisTarget::createBenchmarkRunner( Benchmark::ModeE Mode, const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, BenchmarkRunner::ExecutionModeE ExecutionMode, + unsigned BenchmarkRepeatCount, Benchmark::ResultAggregationModeE ResultAggMode) const { PfmCountersInfo PfmCounters = State.getPfmCounters(); switch (Mode) { @@ -101,7 +102,8 @@ ExegesisTarget::createBenchmarkRunner( "the kernel for real event counts.")); } return createLatencyBenchmarkRunner(State, Mode, BenchmarkPhaseSelector, - ResultAggMode, ExecutionMode); + ResultAggMode, ExecutionMode, + BenchmarkRepeatCount); case Benchmark::Uops: if (BenchmarkPhaseSelector == BenchmarkPhaseSelectorE::Measure && !PfmCounters.UopsCounter && !PfmCounters.IssueCounters) @@ -130,9 +132,11 @@ std::unique_ptr ExegesisTarget::createLatencyBenchmarkRunner( const LLVMState &State, Benchmark::ModeE Mode, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, Benchmark::ResultAggregationModeE ResultAggMode, - BenchmarkRunner::ExecutionModeE ExecutionMode) const { + BenchmarkRunner::ExecutionModeE ExecutionMode, + unsigned BenchmarkRepeatCount) const { return std::make_unique( - State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode); + State, Mode, BenchmarkPhaseSelector, ResultAggMode, ExecutionMode, + BenchmarkRepeatCount); } std::unique_ptr ExegesisTarget::createUopsBenchmarkRunner( diff --git a/llvm/tools/llvm-exegesis/lib/Target.h b/llvm/tools/llvm-exegesis/lib/Target.h index 6de5b3c1065f..c37dd8b70821 100644 --- a/llvm/tools/llvm-exegesis/lib/Target.h +++ b/llvm/tools/llvm-exegesis/lib/Target.h @@ -262,6 +262,7 @@ public: Benchmark::ModeE Mode, const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, BenchmarkRunner::ExecutionModeE ExecutionMode, + unsigned BenchmarkRepeatCount, Benchmark::ResultAggregationModeE ResultAggMode = Benchmark::Min) const; // Returns the ExegesisTarget for the given triple or nullptr if the target @@ -305,7 +306,8 @@ private: const LLVMState &State, Benchmark::ModeE Mode, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, Benchmark::ResultAggregationModeE ResultAggMode, - BenchmarkRunner::ExecutionModeE ExecutionMode) const; + BenchmarkRunner::ExecutionModeE ExecutionMode, + unsigned BenchmarkRepeatCount) const; std::unique_ptr virtual createUopsBenchmarkRunner( const LLVMState &State, BenchmarkPhaseSelectorE BenchmarkPhaseSelector, Benchmark::ResultAggregationModeE ResultAggMode, diff --git a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp index 261335a817d0..148891a18246 100644 --- a/llvm/tools/llvm-exegesis/llvm-exegesis.cpp +++ b/llvm/tools/llvm-exegesis/llvm-exegesis.cpp @@ -262,6 +262,12 @@ static cl::opt ExecutionMode( "allows for the use of memory annotations")), cl::init(BenchmarkRunner::ExecutionModeE::InProcess)); +static cl::opt BenchmarkRepeatCount( + "benchmark-repeat-count", + cl::desc("The number of times to repeat measurements on the benchmark k " + "before aggregating the results"), + cl::cat(BenchmarkOptions), cl::init(30)); + static ExitOnError ExitOnErr("llvm-exegesis error: "); // Helper function that logs the error(s) and exits. @@ -404,8 +410,18 @@ static void runBenchmarkConfigurations( std::optional DumpFile; if (DumpObjectToDisk.getNumOccurrences()) DumpFile = DumpObjectToDisk; - AllResults.emplace_back( - ExitOnErr(Runner.runConfiguration(std::move(RC), DumpFile))); + auto [Err, InstrBenchmark] = + Runner.runConfiguration(std::move(RC), DumpFile); + if (Err) { + // Errors from executing the snippets are fine. + // All other errors are a framework issue and should fail. + if (!Err.isA()) { + llvm::errs() << "llvm-exegesis error: " << toString(std::move(Err)); + exit(1); + } + InstrBenchmark.Error = toString(std::move(Err)); + } + AllResults.push_back(std::move(InstrBenchmark)); } Benchmark &Result = AllResults.front(); @@ -485,7 +501,7 @@ void benchmarkMain() { const std::unique_ptr Runner = ExitOnErr(State.getExegesisTarget().createBenchmarkRunner( BenchmarkMode, State, BenchmarkPhaseSelector, ExecutionMode, - ResultAggMode)); + BenchmarkRepeatCount, ResultAggMode)); if (!Runner) { ExitWithError("cannot create benchmark runner"); } @@ -533,8 +549,10 @@ void benchmarkMain() { for (const auto &Configuration : Configurations) { if (ExecutionMode != BenchmarkRunner::ExecutionModeE::SubProcess && (Configuration.Key.MemoryMappings.size() != 0 || - Configuration.Key.MemoryValues.size() != 0)) - ExitWithError("Memory annotations are only supported in subprocess " + Configuration.Key.MemoryValues.size() != 0 || + Configuration.Key.SnippetAddress != 0)) + ExitWithError("Memory and snippet address annotations are only " + "supported in subprocess " "execution mode"); } } diff --git a/llvm/tools/llvm-ml/llvm-ml.cpp b/llvm/tools/llvm-ml/llvm-ml.cpp index 4d6bd90d13ad..1c713304e4ea 100644 --- a/llvm/tools/llvm-ml/llvm-ml.cpp +++ b/llvm/tools/llvm-ml/llvm-ml.cpp @@ -83,7 +83,7 @@ static Triple GetTriple(StringRef ProgName, opt::InputArgList &Args) { StringRef DefaultBitness = "32"; SmallString<255> Program = ProgName; sys::path::replace_extension(Program, ""); - if (Program.endswith("ml64")) + if (Program.ends_with("ml64")) DefaultBitness = "64"; StringRef TripleName = diff --git a/llvm/tools/llvm-readobj/ELFDumper.cpp b/llvm/tools/llvm-readobj/ELFDumper.cpp index d6d0ea35044a..a9a31ae6c474 100644 --- a/llvm/tools/llvm-readobj/ELFDumper.cpp +++ b/llvm/tools/llvm-readobj/ELFDumper.cpp @@ -2027,6 +2027,18 @@ template void ELFDumper::parseDynamicTable() { uint64_t StringTableSize = 0; std::optional DynSymFromTable; for (const Elf_Dyn &Dyn : dynamic_table()) { + if (Obj.getHeader().e_machine == EM_AARCH64) { + switch (Dyn.d_tag) { + case ELF::DT_AARCH64_AUTH_RELRSZ: + DynRelrRegion.Size = Dyn.getVal(); + DynRelrRegion.SizePrintName = "DT_AARCH64_AUTH_RELRSZ value"; + continue; + case ELF::DT_AARCH64_AUTH_RELRENT: + DynRelrRegion.EntSize = Dyn.getVal(); + DynRelrRegion.EntSizePrintName = "DT_AARCH64_AUTH_RELRENT value"; + continue; + } + } switch (Dyn.d_tag) { case ELF::DT_HASH: HashTable = reinterpret_cast( @@ -2090,10 +2102,12 @@ template void ELFDumper::parseDynamicTable() { break; case ELF::DT_RELR: case ELF::DT_ANDROID_RELR: + case ELF::DT_AARCH64_AUTH_RELR: DynRelrRegion.Addr = toMappedAddr(Dyn.getTag(), Dyn.getPtr()); break; case ELF::DT_RELRSZ: case ELF::DT_ANDROID_RELRSZ: + case ELF::DT_AARCH64_AUTH_RELRSZ: DynRelrRegion.Size = Dyn.getVal(); DynRelrRegion.SizePrintName = Dyn.d_tag == ELF::DT_RELRSZ ? "DT_RELRSZ value" @@ -2101,6 +2115,7 @@ template void ELFDumper::parseDynamicTable() { break; case ELF::DT_RELRENT: case ELF::DT_ANDROID_RELRENT: + case ELF::DT_AARCH64_AUTH_RELRENT: DynRelrRegion.EntSize = Dyn.getVal(); DynRelrRegion.EntSizePrintName = Dyn.d_tag == ELF::DT_RELRENT ? "DT_RELRENT value" @@ -2467,6 +2482,8 @@ std::string ELFDumper::getDynamicEntry(uint64_t Type, case DT_PREINIT_ARRAYSZ: case DT_RELRSZ: case DT_RELRENT: + case DT_AARCH64_AUTH_RELRSZ: + case DT_AARCH64_AUTH_RELRENT: case DT_ANDROID_RELSZ: case DT_ANDROID_RELASZ: return std::to_string(Value) + " (bytes)"; @@ -2564,7 +2581,7 @@ template void ELFDumper::printNeededLibraries() { llvm::sort(Libs); for (StringRef L : Libs) - W.startLine() << L << "\n"; + W.printString(L); } template @@ -3799,9 +3816,12 @@ void GNUELFDumper::printRelRelaReloc(const Relocation &R, } template -static void printRelocHeaderFields(formatted_raw_ostream &OS, unsigned SType) { +static void printRelocHeaderFields(formatted_raw_ostream &OS, unsigned SType, + const typename ELFT::Ehdr &EHeader) { bool IsRela = SType == ELF::SHT_RELA || SType == ELF::SHT_ANDROID_RELA; - bool IsRelr = SType == ELF::SHT_RELR || SType == ELF::SHT_ANDROID_RELR; + bool IsRelr = + SType == ELF::SHT_RELR || SType == ELF::SHT_ANDROID_RELR || + (EHeader.e_machine == EM_AARCH64 && SType == ELF::SHT_AARCH64_AUTH_RELR); if (ELFT::Is64Bits) OS << " "; else @@ -3826,15 +3846,18 @@ void GNUELFDumper::printDynamicRelocHeader(unsigned Type, StringRef Name, uint64_t Offset = Reg.Addr - this->Obj.base(); OS << "\n'" << Name.str().c_str() << "' relocation section at offset 0x" << utohexstr(Offset, /*LowerCase=*/true) << " contains " << Reg.Size << " bytes:\n"; - printRelocHeaderFields(OS, Type); + printRelocHeaderFields(OS, Type, this->Obj.getHeader()); } template -static bool isRelocationSec(const typename ELFT::Shdr &Sec) { +static bool isRelocationSec(const typename ELFT::Shdr &Sec, + const typename ELFT::Ehdr &EHeader) { return Sec.sh_type == ELF::SHT_REL || Sec.sh_type == ELF::SHT_RELA || Sec.sh_type == ELF::SHT_RELR || Sec.sh_type == ELF::SHT_ANDROID_REL || Sec.sh_type == ELF::SHT_ANDROID_RELA || - Sec.sh_type == ELF::SHT_ANDROID_RELR; + Sec.sh_type == ELF::SHT_ANDROID_RELR || + (EHeader.e_machine == EM_AARCH64 && + Sec.sh_type == ELF::SHT_AARCH64_AUTH_RELR); } template void GNUELFDumper::printRelocations() { @@ -3850,8 +3873,10 @@ template void GNUELFDumper::printRelocations() { return RelasOrErr->size(); } - if (!opts::RawRelr && (Sec.sh_type == ELF::SHT_RELR || - Sec.sh_type == ELF::SHT_ANDROID_RELR)) { + if (!opts::RawRelr && + (Sec.sh_type == ELF::SHT_RELR || Sec.sh_type == ELF::SHT_ANDROID_RELR || + (this->Obj.getHeader().e_machine == EM_AARCH64 && + Sec.sh_type == ELF::SHT_AARCH64_AUTH_RELR))) { Expected RelrsOrErr = this->Obj.relrs(Sec); if (!RelrsOrErr) return RelrsOrErr.takeError(); @@ -3863,7 +3888,7 @@ template void GNUELFDumper::printRelocations() { bool HasRelocSections = false; for (const Elf_Shdr &Sec : cantFail(this->Obj.sections())) { - if (!isRelocationSec(Sec)) + if (!isRelocationSec(Sec, this->Obj.getHeader())) continue; HasRelocSections = true; @@ -3880,7 +3905,7 @@ template void GNUELFDumper::printRelocations() { OS << "\nRelocation section '" << Name << "' at offset 0x" << utohexstr(Offset, /*LowerCase=*/true) << " contains " << EntriesNum << " entries:\n"; - printRelocHeaderFields(OS, Sec.sh_type); + printRelocHeaderFields(OS, Sec.sh_type, this->Obj.getHeader()); this->printRelocationsHelper(Sec); } if (!HasRelocSections) @@ -5312,6 +5337,31 @@ static bool printAndroidNote(raw_ostream &OS, uint32_t NoteType, return true; } +template +static bool printAArch64Note(raw_ostream &OS, uint32_t NoteType, + ArrayRef Desc) { + if (NoteType != NT_ARM_TYPE_PAUTH_ABI_TAG) + return false; + + OS << " AArch64 PAuth ABI tag: "; + if (Desc.size() < 16) { + OS << format("", Desc.size()); + return false; + } + + uint64_t Platform = + support::endian::read64(Desc.data() + 0); + uint64_t Version = + support::endian::read64(Desc.data() + 8); + OS << format("platform 0x%" PRIx64 ", version 0x%" PRIx64, Platform, Version); + + if (Desc.size() > 16) + OS << ", additional info 0x" + << toHex(ArrayRef(Desc.data() + 16, Desc.size() - 16)); + + return true; +} + template void GNUELFDumper::printMemtag( const ArrayRef> DynamicEntries, @@ -5711,6 +5761,10 @@ const NoteType AndroidNoteTypes[] = { "NT_ANDROID_TYPE_MEMTAG (Android memory tagging information)"}, }; +const NoteType ARMNoteTypes[] = { + {ELF::NT_ARM_TYPE_PAUTH_ABI_TAG, "NT_ARM_TYPE_PAUTH_ABI_TAG"}, +}; + const NoteType CoreNoteTypes[] = { {ELF::NT_PRSTATUS, "NT_PRSTATUS (prstatus structure)"}, {ELF::NT_FPREGSET, "NT_FPREGSET (floating point registers)"}, @@ -5829,6 +5883,8 @@ StringRef getNoteTypeName(const typename ELFT::Note &Note, unsigned ELFType) { return FindNote(LLVMOMPOFFLOADNoteTypes); if (Name == "Android") return FindNote(AndroidNoteTypes); + if (Name == "ARM") + return FindNote(ARMNoteTypes); if (ELFType == ELF::ET_CORE) return FindNote(CoreNoteTypes); @@ -5984,6 +6040,9 @@ template void GNUELFDumper::printNotes() { } else if (Name == "Android") { if (printAndroidNote(OS, Type, Descriptor)) return Error::success(); + } else if (Name == "ARM") { + if (printAArch64Note(OS, Type, Descriptor)) + return Error::success(); } if (!Descriptor.empty()) { OS << " description data:"; @@ -6176,11 +6235,13 @@ void ELFDumper::forEachRelocationDo( toString(std::move(E))); }; - // SHT_RELR/SHT_ANDROID_RELR sections do not have an associated symbol table. - // For them we should not treat the value of the sh_link field as an index of - // a symbol table. + // SHT_RELR/SHT_ANDROID_RELR/SHT_AARCH64_AUTH_RELR sections do not have an + // associated symbol table. For them we should not treat the value of the + // sh_link field as an index of a symbol table. const Elf_Shdr *SymTab; - if (Sec.sh_type != ELF::SHT_RELR && Sec.sh_type != ELF::SHT_ANDROID_RELR) { + if (Sec.sh_type != ELF::SHT_RELR && Sec.sh_type != ELF::SHT_ANDROID_RELR && + !(Obj.getHeader().e_machine == EM_AARCH64 && + Sec.sh_type == ELF::SHT_AARCH64_AUTH_RELR)) { Expected SymTabOrErr = Obj.getSection(Sec.sh_link); if (!SymTabOrErr) { Warn(SymTabOrErr.takeError(), "unable to locate a symbol table for"); @@ -6208,6 +6269,10 @@ void ELFDumper::forEachRelocationDo( Warn(RangeOrErr.takeError()); } break; + case ELF::SHT_AARCH64_AUTH_RELR: + if (Obj.getHeader().e_machine != EM_AARCH64) + break; + [[fallthrough]]; case ELF::SHT_RELR: case ELF::SHT_ANDROID_RELR: { Expected RangeOrErr = Obj.relrs(Sec); @@ -6904,7 +6969,7 @@ template void LLVMELFDumper::printRelocations() { ListScope D(W, "Relocations"); for (const Elf_Shdr &Sec : cantFail(this->Obj.sections())) { - if (!isRelocationSec(Sec)) + if (!isRelocationSec(Sec, this->Obj.getHeader())) continue; StringRef Name = this->getPrintableSectionName(Sec); @@ -7545,6 +7610,29 @@ static bool printAndroidNoteLLVMStyle(uint32_t NoteType, ArrayRef Desc, return true; } +template +static bool printAarch64NoteLLVMStyle(uint32_t NoteType, ArrayRef Desc, + ScopedPrinter &W) { + if (NoteType != NT_ARM_TYPE_PAUTH_ABI_TAG) + return false; + + if (Desc.size() < 16) + return false; + + uint64_t platform = + support::endian::read64(Desc.data() + 0); + uint64_t version = + support::endian::read64(Desc.data() + 8); + W.printNumber("Platform", platform); + W.printNumber("Version", version); + + if (Desc.size() > 16) + W.printString("Additional info", + toHex(ArrayRef(Desc.data() + 16, Desc.size() - 16))); + + return true; +} + template void LLVMELFDumper::printMemtag( const ArrayRef> DynamicEntries, @@ -7681,6 +7769,9 @@ template void LLVMELFDumper::printNotes() { } else if (Name == "Android") { if (printAndroidNoteLLVMStyle(Type, Descriptor, W)) return Error::success(); + } else if (Name == "ARM") { + if (printAarch64NoteLLVMStyle(Type, Descriptor, W)) + return Error::success(); } if (!Descriptor.empty()) { W.printBinaryBlock("Description data", Descriptor); diff --git a/llvm/tools/llvm-readtapi/llvm-readtapi.cpp b/llvm/tools/llvm-readtapi/llvm-readtapi.cpp index cb2b36072a55..64eff70dbfc7 100644 --- a/llvm/tools/llvm-readtapi/llvm-readtapi.cpp +++ b/llvm/tools/llvm-readtapi/llvm-readtapi.cpp @@ -56,30 +56,31 @@ public: } }; +struct Context { + std::vector Inputs; + std::unique_ptr OutStream; + FileType WriteFT = FileType::TBD_V5; + bool Compact = false; + Architecture Arch = AK_unknown; +}; + // Use unique exit code to differentiate failures not directly caused from // TextAPI operations. This is used for wrapping `compare` operations in // automation and scripting. const int NON_TAPI_EXIT_CODE = 2; const std::string TOOLNAME = "llvm-readtapi"; ExitOnError ExitOnErr; +} // anonymous namespace // Handle error reporting in cases where `ExitOnError` is not used. -void reportError(Twine Message, int ExitCode = EXIT_FAILURE) { +static void reportError(Twine Message, int ExitCode = EXIT_FAILURE) { errs() << TOOLNAME << ": error: " << Message << "\n"; errs().flush(); exit(ExitCode); } -struct Context { - std::vector Inputs; - std::unique_ptr OutStream; - FileType WriteFT = FileType::TBD_V5; - bool Compact = false; - Architecture Arch = AK_unknown; -}; - -std::unique_ptr getInterfaceFile(const StringRef Filename, - bool ResetBanner = true) { +static std::unique_ptr +getInterfaceFile(const StringRef Filename, bool ResetBanner = true) { ExitOnErr.setBanner(TOOLNAME + ": error: '" + Filename.str() + "' "); ErrorOr> BufferOrErr = MemoryBuffer::getFile(Filename); @@ -94,7 +95,7 @@ std::unique_ptr getInterfaceFile(const StringRef Filename, return std::move(*IF); } -bool handleCompareAction(const Context &Ctx) { +static bool handleCompareAction(const Context &Ctx) { if (Ctx.Inputs.size() != 2) reportError("compare only supports two input files", /*ExitCode=*/NON_TAPI_EXIT_CODE); @@ -109,8 +110,8 @@ bool handleCompareAction(const Context &Ctx) { return DiffEngine(LeftIF.get(), RightIF.get()).compareFiles(OS); } -bool handleWriteAction(const Context &Ctx, - std::unique_ptr Out = nullptr) { +static bool handleWriteAction(const Context &Ctx, + std::unique_ptr Out = nullptr) { if (!Out) { if (Ctx.Inputs.size() != 1) reportError("write only supports one input file"); @@ -121,7 +122,7 @@ bool handleWriteAction(const Context &Ctx, return EXIT_SUCCESS; } -bool handleMergeAction(const Context &Ctx) { +static bool handleMergeAction(const Context &Ctx) { if (Ctx.Inputs.size() < 2) reportError("merge requires at least two input files"); @@ -144,8 +145,8 @@ bool handleMergeAction(const Context &Ctx) { using IFOperation = std::function>( const llvm::MachO::InterfaceFile &, Architecture)>; -bool handleSingleFileAction(const Context &Ctx, const StringRef Action, - IFOperation act) { +static bool handleSingleFileAction(const Context &Ctx, const StringRef Action, + IFOperation act) { if (Ctx.Inputs.size() != 1) reportError(Action + " only supports one input file"); if (Ctx.Arch == AK_unknown) @@ -159,8 +160,6 @@ bool handleSingleFileAction(const Context &Ctx, const StringRef Action, return handleWriteAction(Ctx, std::move(*OutIF)); } -} // anonymous namespace - int main(int Argc, char **Argv) { InitLLVM X(Argc, Argv); BumpPtrAllocator A; diff --git a/llvm/tools/llvm-shlib/CMakeLists.txt b/llvm/tools/llvm-shlib/CMakeLists.txt index 8ace190b28f7..64d6f631ffad 100644 --- a/llvm/tools/llvm-shlib/CMakeLists.txt +++ b/llvm/tools/llvm-shlib/CMakeLists.txt @@ -14,6 +14,9 @@ if(LLVM_BUILD_LLVM_DYLIB) if(MSVC) message(FATAL_ERROR "Generating libLLVM is not supported on MSVC") endif() + if(ZOS) + message(FATAL_ERROR "Generating libLLVM is not supported on z/OS") + endif() llvm_map_components_to_libnames(LIB_NAMES ${LLVM_DYLIB_COMPONENTS}) @@ -33,16 +36,9 @@ if(LLVM_BUILD_LLVM_DYLIB) add_llvm_library(LLVM SHARED DISABLE_LLVM_LINK_LLVM_DYLIB SONAME ${INSTALL_WITH_TOOLCHAIN} ${SOURCES}) list(REMOVE_DUPLICATES LIB_NAMES) - if((MINGW) OR (HAIKU) - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "Linux") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "GNU") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "FreeBSD") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "NetBSD") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "OpenBSD") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "DragonFly") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "Fuchsia") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "Android") - OR ("${CMAKE_SYSTEM_NAME}" STREQUAL "SunOS")) # FIXME: It should be "GNU ld for elf" + if("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") + set(LIB_NAMES -Wl,-all_load ${LIB_NAMES}) + else() configure_file( ${CMAKE_CURRENT_SOURCE_DIR}/simple_version_script.map.in ${LLVM_LIBRARY_DIR}/tools/llvm-shlib/simple_version_script.map) @@ -60,8 +56,6 @@ if(LLVM_BUILD_LLVM_DYLIB) # inside and outside libLLVM.so. target_link_options(LLVM PRIVATE LINKER:-Bsymbolic-functions) endif() - elseif("${CMAKE_SYSTEM_NAME}" STREQUAL "Darwin") - set(LIB_NAMES -Wl,-all_load ${LIB_NAMES}) endif() target_link_libraries(LLVM PRIVATE ${LIB_NAMES}) diff --git a/llvm/tools/obj2yaml/xcoff2yaml.cpp b/llvm/tools/obj2yaml/xcoff2yaml.cpp index 882c41049601..f7c2bae74798 100644 --- a/llvm/tools/obj2yaml/xcoff2yaml.cpp +++ b/llvm/tools/obj2yaml/xcoff2yaml.cpp @@ -25,10 +25,30 @@ class XCOFFDumper { template Error dumpSections(ArrayRef Sections); + // Dump auxiliary symbols. + Error dumpFileAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef); + Error dumpStatAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef); + Error dumpBlockAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef); + Error dumpDwarfAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef); + Error dumpAuxSyms(XCOFFYAML::Symbol &Sym, const XCOFFSymbolRef &SymbolEntRef); + void dumpFuncAuxSym(XCOFFYAML::Symbol &Sym, const uintptr_t AuxAddress); + void dumpExpAuxSym(XCOFFYAML::Symbol &Sym, const uintptr_t AuxAddress); + void dumpCscetAuxSym(XCOFFYAML::Symbol &Sym, + const object::XCOFFCsectAuxRef &AuxEntPtr); + public: XCOFFDumper(const object::XCOFFObjectFile &obj) : Obj(obj) {} Error dump(); XCOFFYAML::Object &getYAMLObj() { return YAMLObj; } + + template const T *getAuxEntPtr(uintptr_t AuxAddress) { + Obj.checkSymbolEntryPointer(AuxAddress); + return reinterpret_cast(AuxAddress); + } }; } // namespace @@ -106,6 +126,210 @@ Error XCOFFDumper::dumpSections(ArrayRef Sections) { return Error::success(); } +Error XCOFFDumper::dumpFileAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef) { + for (uint8_t I = 1; I <= Sym.NumberOfAuxEntries; ++I) { + uintptr_t AuxAddress = XCOFFObjectFile::getAdvancedSymbolEntryAddress( + SymbolEntRef.getEntryAddress(), I); + const XCOFFFileAuxEnt *FileAuxEntPtr = + getAuxEntPtr(AuxAddress); + auto FileNameOrError = Obj.getCFileName(FileAuxEntPtr); + if (!FileNameOrError) + return FileNameOrError.takeError(); + + XCOFFYAML::FileAuxEnt FileAuxSym; + FileAuxSym.FileNameOrString = FileNameOrError.get(); + FileAuxSym.FileStringType = FileAuxEntPtr->Type; + Sym.AuxEntries.push_back( + std::make_unique(FileAuxSym)); + } + return Error::success(); +} + +Error XCOFFDumper::dumpStatAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef) { + if (Sym.NumberOfAuxEntries != 1) { + uint32_t SymbolIndex = Obj.getSymbolIndex(SymbolEntRef.getEntryAddress()); + return createError("failed to parse symbol \"" + Sym.SymbolName + + "\" with index of " + Twine(SymbolIndex) + + ": expected 1 aux symbol for C_STAT, while got " + + Twine(static_cast(*Sym.NumberOfAuxEntries))); + } + + const XCOFFSectAuxEntForStat *AuxEntPtr = + getAuxEntPtr( + XCOFFObjectFile::getAdvancedSymbolEntryAddress( + SymbolEntRef.getEntryAddress(), 1)); + XCOFFYAML::SectAuxEntForStat StatAuxSym; + StatAuxSym.SectionLength = AuxEntPtr->SectionLength; + StatAuxSym.NumberOfLineNum = AuxEntPtr->NumberOfLineNum; + StatAuxSym.NumberOfRelocEnt = AuxEntPtr->NumberOfRelocEnt; + Sym.AuxEntries.push_back( + std::make_unique(StatAuxSym)); + return Error::success(); +} + +void XCOFFDumper::dumpFuncAuxSym(XCOFFYAML::Symbol &Sym, + const uintptr_t AuxAddress) { + XCOFFYAML::FunctionAuxEnt FunAuxSym; + + if (Obj.is64Bit()) { + const XCOFFFunctionAuxEnt64 *AuxEntPtr = + getAuxEntPtr(AuxAddress); + FunAuxSym.PtrToLineNum = AuxEntPtr->PtrToLineNum; + FunAuxSym.SizeOfFunction = AuxEntPtr->SizeOfFunction; + FunAuxSym.SymIdxOfNextBeyond = AuxEntPtr->SymIdxOfNextBeyond; + } else { + const XCOFFFunctionAuxEnt32 *AuxEntPtr = + getAuxEntPtr(AuxAddress); + FunAuxSym.OffsetToExceptionTbl = AuxEntPtr->OffsetToExceptionTbl; + FunAuxSym.PtrToLineNum = AuxEntPtr->PtrToLineNum; + FunAuxSym.SizeOfFunction = AuxEntPtr->SizeOfFunction; + FunAuxSym.SymIdxOfNextBeyond = AuxEntPtr->SymIdxOfNextBeyond; + } + + Sym.AuxEntries.push_back( + std::make_unique(FunAuxSym)); +} + +void XCOFFDumper::dumpExpAuxSym(XCOFFYAML::Symbol &Sym, + const uintptr_t AuxAddress) { + const XCOFFExceptionAuxEnt *AuxEntPtr = + getAuxEntPtr(AuxAddress); + XCOFFYAML::ExcpetionAuxEnt ExceptAuxSym; + ExceptAuxSym.OffsetToExceptionTbl = AuxEntPtr->OffsetToExceptionTbl; + ExceptAuxSym.SizeOfFunction = AuxEntPtr->SizeOfFunction; + ExceptAuxSym.SymIdxOfNextBeyond = AuxEntPtr->SymIdxOfNextBeyond; + Sym.AuxEntries.push_back( + std::make_unique(ExceptAuxSym)); +} + +void XCOFFDumper::dumpCscetAuxSym(XCOFFYAML::Symbol &Sym, + const object::XCOFFCsectAuxRef &AuxEntPtr) { + XCOFFYAML::CsectAuxEnt CsectAuxSym; + CsectAuxSym.ParameterHashIndex = AuxEntPtr.getParameterHashIndex(); + CsectAuxSym.TypeChkSectNum = AuxEntPtr.getTypeChkSectNum(); + CsectAuxSym.SymbolAlignmentAndType = AuxEntPtr.getSymbolAlignmentAndType(); + CsectAuxSym.StorageMappingClass = AuxEntPtr.getStorageMappingClass(); + + if (Obj.is64Bit()) { + CsectAuxSym.SectionOrLengthLo = + static_cast(AuxEntPtr.getSectionOrLength64()); + CsectAuxSym.SectionOrLengthHi = + static_cast(AuxEntPtr.getSectionOrLength64() >> 32); + } else { + CsectAuxSym.SectionOrLength = AuxEntPtr.getSectionOrLength32(); + CsectAuxSym.StabInfoIndex = AuxEntPtr.getStabInfoIndex32(); + CsectAuxSym.StabSectNum = AuxEntPtr.getStabSectNum32(); + } + + Sym.AuxEntries.push_back( + std::make_unique(CsectAuxSym)); +} + +Error XCOFFDumper::dumpAuxSyms(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef) { + auto ErrOrCsectAuxRef = SymbolEntRef.getXCOFFCsectAuxRef(); + if (!ErrOrCsectAuxRef) + return ErrOrCsectAuxRef.takeError(); + XCOFFCsectAuxRef CsectAuxRef = ErrOrCsectAuxRef.get(); + + for (uint8_t I = 1; I <= Sym.NumberOfAuxEntries; ++I) { + + if (I == Sym.NumberOfAuxEntries && !Obj.is64Bit()) { + dumpCscetAuxSym(Sym, CsectAuxRef); + return Error::success(); + } + + uintptr_t AuxAddress = XCOFFObjectFile::getAdvancedSymbolEntryAddress( + SymbolEntRef.getEntryAddress(), I); + + if (Obj.is64Bit()) { + XCOFF::SymbolAuxType Type = *Obj.getSymbolAuxType(AuxAddress); + if (Type == XCOFF::SymbolAuxType::AUX_CSECT) + dumpCscetAuxSym(Sym, CsectAuxRef); + else if (Type == XCOFF::SymbolAuxType::AUX_FCN) + dumpFuncAuxSym(Sym, AuxAddress); + else if (Type == XCOFF::SymbolAuxType::AUX_EXCEPT) + dumpExpAuxSym(Sym, AuxAddress); + else { + uint32_t SymbolIndex = + Obj.getSymbolIndex(SymbolEntRef.getEntryAddress()); + return createError("failed to parse symbol \"" + Sym.SymbolName + + "\" with index of " + Twine(SymbolIndex) + + ": invalid auxiliary symbol type: " + + Twine(static_cast(Type))); + } + + } else + dumpFuncAuxSym(Sym, AuxAddress); + } + + return Error::success(); +} + +Error XCOFFDumper::dumpBlockAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef) { + if (Sym.NumberOfAuxEntries != 1) { + uint32_t SymbolIndex = Obj.getSymbolIndex(SymbolEntRef.getEntryAddress()); + return createError( + "failed to parse symbol \"" + Sym.SymbolName + "\" with index of " + + Twine(SymbolIndex) + + ": expected 1 aux symbol for C_BLOCK or C_FCN, while got " + + Twine(static_cast(*Sym.NumberOfAuxEntries))); + } + + uintptr_t AuxAddress = XCOFFObjectFile::getAdvancedSymbolEntryAddress( + SymbolEntRef.getEntryAddress(), 1); + XCOFFYAML::BlockAuxEnt BlockAuxSym; + + if (Obj.is64Bit()) { + const XCOFFBlockAuxEnt64 *AuxEntPtr = + getAuxEntPtr(AuxAddress); + BlockAuxSym.LineNum = AuxEntPtr->LineNum; + } else { + const XCOFFBlockAuxEnt32 *AuxEntPtr = + getAuxEntPtr(AuxAddress); + BlockAuxSym.LineNumLo = AuxEntPtr->LineNumLo; + BlockAuxSym.LineNumHi = AuxEntPtr->LineNumHi; + } + + Sym.AuxEntries.push_back( + std::make_unique(BlockAuxSym)); + return Error::success(); +} + +Error XCOFFDumper::dumpDwarfAuxSym(XCOFFYAML::Symbol &Sym, + const XCOFFSymbolRef &SymbolEntRef) { + if (Sym.NumberOfAuxEntries != 1) { + uint32_t SymbolIndex = Obj.getSymbolIndex(SymbolEntRef.getEntryAddress()); + return createError("failed to parse symbol \"" + Sym.SymbolName + + "\" with index of " + Twine(SymbolIndex) + + ": expected 1 aux symbol for C_DWARF, while got " + + Twine(static_cast(*Sym.NumberOfAuxEntries))); + } + + uintptr_t AuxAddress = XCOFFObjectFile::getAdvancedSymbolEntryAddress( + SymbolEntRef.getEntryAddress(), 1); + XCOFFYAML::SectAuxEntForDWARF DwarfAuxSym; + + if (Obj.is64Bit()) { + const XCOFFSectAuxEntForDWARF64 *AuxEntPtr = + getAuxEntPtr(AuxAddress); + DwarfAuxSym.LengthOfSectionPortion = AuxEntPtr->LengthOfSectionPortion; + DwarfAuxSym.NumberOfRelocEnt = AuxEntPtr->NumberOfRelocEnt; + } else { + const XCOFFSectAuxEntForDWARF32 *AuxEntPtr = + getAuxEntPtr(AuxAddress); + DwarfAuxSym.LengthOfSectionPortion = AuxEntPtr->LengthOfSectionPortion; + DwarfAuxSym.NumberOfRelocEnt = AuxEntPtr->NumberOfRelocEnt; + } + + Sym.AuxEntries.push_back( + std::make_unique(DwarfAuxSym)); + return Error::success(); +} + Error XCOFFDumper::dumpSymbols() { std::vector &Symbols = YAMLObj.Symbols; @@ -133,6 +357,36 @@ Error XCOFFDumper::dumpSymbols() { Sym.StorageClass = SymbolEntRef.getStorageClass(); Sym.NumberOfAuxEntries = SymbolEntRef.getNumberOfAuxEntries(); + if (Sym.NumberOfAuxEntries) { + switch (Sym.StorageClass) { + case XCOFF::C_FILE: + if (Error E = dumpFileAuxSym(Sym, SymbolEntRef)) + return E; + break; + case XCOFF::C_STAT: + if (Error E = dumpStatAuxSym(Sym, SymbolEntRef)) + return E; + break; + case XCOFF::C_EXT: + case XCOFF::C_WEAKEXT: + case XCOFF::C_HIDEXT: + if (Error E = dumpAuxSyms(Sym, SymbolEntRef)) + return E; + break; + case XCOFF::C_BLOCK: + case XCOFF::C_FCN: + if (Error E = dumpBlockAuxSym(Sym, SymbolEntRef)) + return E; + break; + case XCOFF::C_DWARF: + if (Error E = dumpDwarfAuxSym(Sym, SymbolEntRef)) + return E; + break; + default: + break; + } + } + Symbols.push_back(std::move(Sym)); } diff --git a/llvm/tools/opt/opt.cpp b/llvm/tools/opt/opt.cpp index 5e5e5ce233f3..50b36dc74267 100644 --- a/llvm/tools/opt/opt.cpp +++ b/llvm/tools/opt/opt.cpp @@ -339,7 +339,8 @@ static bool shouldPinPassToLegacyPM(StringRef Pass) { "nvptx-", "mips-", "lanai-", "hexagon-", "bpf-", "avr-", "thumb2-", "arm-", "si-", "gcn-", "amdgpu-", "aarch64-", "amdgcn-", "polly-", "riscv-", "dxil-"}; - std::vector PassNameContain = {"ehprepare"}; + // TODO: remove "ehprepare" + std::vector PassNameContain = {"-eh-prepare", "ehprepare"}; std::vector PassNameExact = { "safe-stack", "cost-model", diff --git a/llvm/unittests/Analysis/ValueLatticeTest.cpp b/llvm/unittests/Analysis/ValueLatticeTest.cpp index ae221811f3fb..b456f286e0d9 100644 --- a/llvm/unittests/Analysis/ValueLatticeTest.cpp +++ b/llvm/unittests/Analysis/ValueLatticeTest.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "llvm/Analysis/ValueLattice.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/IR/ConstantRange.h" #include "llvm/IR/Constants.h" #include "llvm/IR/IRBuilder.h" diff --git a/llvm/unittests/Analysis/VectorFunctionABITest.cpp b/llvm/unittests/Analysis/VectorFunctionABITest.cpp index 81c1807cdcaa..201dd1127ef2 100644 --- a/llvm/unittests/Analysis/VectorFunctionABITest.cpp +++ b/llvm/unittests/Analysis/VectorFunctionABITest.cpp @@ -1,4 +1,4 @@ -//===------- VectorFunctionABITest.cpp - VFABI Unittests ---------===// +//===------- VectorFunctionABITest.cpp - VFABI unit tests ---------===// // // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. // See https://llvm.org/LICENSE.txt for license information. @@ -6,6 +6,7 @@ // //===----------------------------------------------------------------------===// +#include "llvm/ADT/StringRef.h" #include "llvm/Analysis/VectorUtils.h" #include "llvm/AsmParser/Parser.h" #include "llvm/IR/InstIterator.h" @@ -14,24 +15,31 @@ using namespace llvm; namespace { -// Test fixture needed that holds the veariables needed by the parser. +/// Perform tests against VFABI Rules. `invokeParser` creates a VFInfo object +/// and a scalar FunctionType, which are used by tests to check that: +/// 1. The scalar and vector names are correct. +/// 2. The number of parameters from the parsed mangled name matches the number +/// of arguments in the scalar function passed as FunctionType string. +/// 3. The number of vector parameters and their types match the values +/// specified in the test. +/// On masked functions it also checks that the last parameter is a mask (ie, +/// GlobalPredicate). +/// 4. The vector function is correctly found to have a mask. +/// class VFABIParserTest : public ::testing::Test { private: // Parser output. VFInfo Info; - // Reset the data needed for the test. - void reset(const StringRef Name, const StringRef IRType) { + /// Reset the data needed for the test. + void reset(const StringRef ScalarFTyStr) { M = parseAssemblyString("declare void @dummy()", Err, Ctx); EXPECT_NE(M.get(), nullptr) << "Loading an invalid module.\n " << Err.getMessage() << "\n"; - Type *Ty = parseType(IRType, Err, *(M.get())); - FTy = dyn_cast(Ty); - EXPECT_NE(FTy, nullptr) << "Invalid function type string: " << IRType - << "\n" - << Err.getMessage() << "\n"; - F = M->getOrInsertFunction(Name, FTy); - EXPECT_NE(F.getCallee(), nullptr) - << "The function must be present in the module\n"; + Type *Ty = parseType(ScalarFTyStr, Err, *(M.get())); + ScalarFTy = dyn_cast(Ty); + EXPECT_NE(ScalarFTy, nullptr) + << "Invalid function type string: " << ScalarFTyStr << "\n" + << Err.getMessage() << "\n"; // Reset the VFInfo Info = VFInfo(); } @@ -40,69 +48,48 @@ private: LLVMContext Ctx; SMDiagnostic Err; std::unique_ptr M; - FunctionType *FTy; - FunctionCallee F; + FunctionType *ScalarFTy = nullptr; protected: - // Referencies to the parser output field. + // References to the parser output field. ElementCount &VF = Info.Shape.VF; VFISAKind &ISA = Info.ISA; + /// Parameters for the vectorized function SmallVector &Parameters = Info.Shape.Parameters; std::string &ScalarName = Info.ScalarName; std::string &VectorName = Info.VectorName; - // Invoke the parser. We need to make sure that a function exist in - // the module because the parser fails if such function don't - // exists. Every time this method is invoked the state of the test - // is reset. - // - // \p MangledName -> the string the parser has to demangle. - // - // \p VectorName -> optional vector name that the method needs to - // use to create the function in the module if it differs from the - // standard mangled name. - // - // \p IRType -> FunctionType string to be used for the signature of - // the vector function. The correct signature is needed by the - // parser only for scalable functions. For the sake of testing, the - // generic fixed-length case can use as signature `void()`. - // + + /// Invoke the parser. Every time this method is invoked the state of the test + /// is reset. + /// + /// \p MangledName string the parser has to demangle. + /// + /// \p ScalarFTyStr FunctionType string to get the signature of the scalar + /// function, which is used by `tryDemangleForVFABI` to check for the number + /// of arguments on scalable vectors, and by `matchParameters` to perform some + /// additional checking in the tests in this file. bool invokeParser(const StringRef MangledName, - const StringRef ScalarName = "", - const StringRef IRType = "void()") { - StringRef Name = MangledName; - if (!ScalarName.empty()) - Name = ScalarName; - // Reset the VFInfo and the Module to be able to invoke - // `invokeParser` multiple times in the same test. - reset(Name, IRType); - - // Fake the arguments to the CallInst. - SmallVector Args; - for (Type *ParamTy : FTy->params()) { - Args.push_back(Constant::getNullValue(ParamTy->getScalarType())); - } - std::unique_ptr CI(CallInst::Create(F, Args)); - const auto OptInfo = - VFABI::tryDemangleForVFABI(MangledName, CI->getFunctionType()); - if (OptInfo) { + const StringRef ScalarFTyStr = "void()") { + // Reset the VFInfo to be able to call `invokeParser` multiple times in + // the same test. + reset(ScalarFTyStr); + + const auto OptInfo = VFABI::tryDemangleForVFABI(MangledName, ScalarFTy); + if (OptInfo) Info = *OptInfo; - return true; - } - return false; + return OptInfo.has_value(); } - // Checks that 1. the last Parameter in the Shape is of type - // VFParamKind::GlobalPredicate and 2. it is the only one of such - // type. - bool IsMasked() const { - const auto NGlobalPreds = - std::count_if(Info.Shape.Parameters.begin(), - Info.Shape.Parameters.end(), [](const VFParameter PK) { - return PK.ParamKind == VFParamKind::GlobalPredicate; - }); - return NGlobalPreds == 1 && Info.Shape.Parameters.back().ParamKind == - VFParamKind::GlobalPredicate; + /// Returns whether the parsed function contains a mask. + bool isMasked() const { return Info.isMasked(); } + + /// Check if the number of vectorized parameters matches the scalar ones. This + /// requires a correct scalar FunctionType string to be fed to the + /// 'invokeParser'. Mask parameters that are only required by the vector + /// function call are ignored. + bool matchParametersNum() { + return (Parameters.size() - isMasked()) == ScalarFTy->getNumParams(); } }; } // unnamed namespace @@ -132,47 +119,74 @@ TEST_F(VFABIParserTest, OnlyValidNames) { EXPECT_FALSE(invokeParser("_ZGVnN2v_")); // Missing _ separator. EXPECT_FALSE(invokeParser("_ZGVnN2vfoo")); - // Missing . Using `fakename` because the string being - // parsed is not a valid function name that `invokeParser` can add. - EXPECT_FALSE(invokeParser("_ZGVnN2v_foo()", "fakename")); - // Unterminated name. Using `fakename` because the string being - // parsed is not a valid function name that `invokeParser` can add. - EXPECT_FALSE(invokeParser("_ZGVnN2v_foo(bar", "fakename")); + // Missing . + EXPECT_FALSE(invokeParser("_ZGVnN2v_foo()")); + // Unterminated name. + EXPECT_FALSE(invokeParser("_ZGVnN2v_foo(bar")); } TEST_F(VFABIParserTest, ParamListParsing) { - EXPECT_TRUE(invokeParser("_ZGVnN2vl16Ls32R3l_foo")); + EXPECT_TRUE( + invokeParser("_ZGVnN2vl16Ls32R3l_foo", "void(i32, i32, i32, ptr, i32)")); + EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_EQ(false, isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; EXPECT_EQ(Parameters.size(), (unsigned)5); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector, 0})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_Linear, 16})); EXPECT_EQ(Parameters[2], VFParameter({2, VFParamKind::OMP_LinearValPos, 32})); EXPECT_EQ(Parameters[3], VFParameter({3, VFParamKind::OMP_LinearRef, 3})); EXPECT_EQ(Parameters[4], VFParameter({4, VFParamKind::OMP_Linear, 1})); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "_ZGVnN2vl16Ls32R3l_foo"); } TEST_F(VFABIParserTest, ScalarNameAndVectorName_01) { - EXPECT_TRUE(invokeParser("_ZGVnM2v_sin")); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "_ZGVnM2v_sin"); + EXPECT_TRUE(invokeParser("_ZGVnM2v_foo(vector_foo)", "void(i32)")); + EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_EQ(true, isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ScalarNameAndVectorName_02) { - EXPECT_TRUE(invokeParser("_ZGVnM2v_sin(UserFunc)", "UserFunc")); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "UserFunc"); + EXPECT_TRUE(invokeParser("_ZGVnM2v_foo(vector_foo)", "void(i32)")); + EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_EQ(true, isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ScalarNameAndVectorName_03) { - EXPECT_TRUE(invokeParser("_ZGVnM2v___sin_sin_sin")); - EXPECT_EQ(ScalarName, "__sin_sin_sin"); - EXPECT_EQ(VectorName, "_ZGVnM2v___sin_sin_sin"); + EXPECT_TRUE( + invokeParser("_ZGVnM2v___foo_bar_abc(fooBarAbcVec)", "void(i32)")); + EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_EQ(true, isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(ScalarName, "__foo_bar_abc"); + EXPECT_EQ(VectorName, "fooBarAbcVec"); +} + +TEST_F(VFABIParserTest, ScalarNameOnly) { + EXPECT_TRUE(invokeParser("_ZGVnM2v___foo_bar_abc")); + EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_EQ(true, isMasked()); + EXPECT_EQ(ScalarName, "__foo_bar_abc"); + // no vector name specified (as it's optional), so it should have the entire + // mangled name. + EXPECT_EQ(VectorName, "_ZGVnM2v___foo_bar_abc"); } TEST_F(VFABIParserTest, Parse) { - EXPECT_TRUE(invokeParser("_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_FALSE(IsMasked()); + EXPECT_TRUE( + invokeParser("_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000_foo", + "void(i32, i32, i32, i32, ptr, i32, i32, i32, ptr)")); EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)9); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector, 0})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_LinearPos, 2})); @@ -183,75 +197,90 @@ TEST_F(VFABIParserTest, Parse) { EXPECT_EQ(Parameters[6], VFParameter({6, VFParamKind::OMP_LinearVal, 10})); EXPECT_EQ(Parameters[7], VFParameter({7, VFParamKind::OMP_LinearUVal, 100})); EXPECT_EQ(Parameters[8], VFParameter({8, VFParamKind::OMP_LinearRef, 1000})); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000_sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000_foo"); } TEST_F(VFABIParserTest, ParseVectorName) { - EXPECT_TRUE(invokeParser("_ZGVnN2v_sin(my_v_sin)", "my_v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_FALSE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVnN2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)1); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector, 0})); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "my_v_sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, LinearWithCompileTimeNegativeStep) { - EXPECT_TRUE(invokeParser("_ZGVnN2ln1Ln10Un100Rn1000_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_FALSE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVnN2ln1Ln10Un100Rn1000_foo(vector_foo)", + "void(i32, i32, i32, ptr)")); EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)4); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::OMP_Linear, -1})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_LinearVal, -10})); EXPECT_EQ(Parameters[2], VFParameter({2, VFParamKind::OMP_LinearUVal, -100})); EXPECT_EQ(Parameters[3], VFParameter({3, VFParamKind::OMP_LinearRef, -1000})); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "_ZGVnN2ln1Ln10Un100Rn1000_sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseScalableSVE) { - EXPECT_TRUE(invokeParser("_ZGVsMxv_sin(custom_vg)", "sin", "i32(i32)")); - EXPECT_EQ(VF, ElementCount::getScalable(4)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVsMxv_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::SVE); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "custom_vg"); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getScalable(4)); + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseFixedWidthSVE) { - EXPECT_TRUE(invokeParser("_ZGVsM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVsM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::SVE); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "_ZGVsM2v_sin"); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, NotAVectorFunctionABIName) { // Vector names should start with `_ZGV`. - EXPECT_FALSE(invokeParser("ZGVnN2v_sin")); + EXPECT_FALSE(invokeParser("ZGVnN2v_foo")); } TEST_F(VFABIParserTest, LinearWithRuntimeStep) { - EXPECT_FALSE(invokeParser("_ZGVnN2ls_sin")) + EXPECT_FALSE(invokeParser("_ZGVnN2ls_foo")) << "A number should be present after \"ls\"."; - EXPECT_TRUE(invokeParser("_ZGVnN2ls2_sin")); - EXPECT_FALSE(invokeParser("_ZGVnN2Rs_sin")) + EXPECT_TRUE(invokeParser("_ZGVnN2ls2_foo")); + EXPECT_FALSE(invokeParser("_ZGVnN2Rs_foo")) << "A number should be present after \"Rs\"."; - EXPECT_TRUE(invokeParser("_ZGVnN2Rs4_sin")); - EXPECT_FALSE(invokeParser("_ZGVnN2Ls_sin")) + EXPECT_TRUE(invokeParser("_ZGVnN2Rs4_foo")); + EXPECT_FALSE(invokeParser("_ZGVnN2Ls_foo")) << "A number should be present after \"Ls\"."; - EXPECT_TRUE(invokeParser("_ZGVnN2Ls6_sin")); - EXPECT_FALSE(invokeParser("_ZGVnN2Us_sin")) + EXPECT_TRUE(invokeParser("_ZGVnN2Ls6_foo")); + EXPECT_FALSE(invokeParser("_ZGVnN2Us_foo")) << "A number should be present after \"Us\"."; - EXPECT_TRUE(invokeParser("_ZGVnN2Us8_sin")); + EXPECT_TRUE(invokeParser("_ZGVnN2Us8_foo")); } TEST_F(VFABIParserTest, LinearWithoutCompileTime) { - EXPECT_TRUE(invokeParser("_ZGVnN3lLRUlnLnRnUn_sin")); + EXPECT_TRUE(invokeParser("_ZGVnN3lLRUlnLnRnUn_foo(vector_foo)", + "void(i32, i32, ptr, i32, i32, i32, ptr, i32)")); + EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; EXPECT_EQ(Parameters.size(), (unsigned)8); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::OMP_Linear, 1})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_LinearVal, 1})); @@ -261,80 +290,73 @@ TEST_F(VFABIParserTest, LinearWithoutCompileTime) { EXPECT_EQ(Parameters[5], VFParameter({5, VFParamKind::OMP_LinearVal, -1})); EXPECT_EQ(Parameters[6], VFParameter({6, VFParamKind::OMP_LinearRef, -1})); EXPECT_EQ(Parameters[7], VFParameter({7, VFParamKind::OMP_LinearUVal, -1})); -} - -TEST_F(VFABIParserTest, ISA) { - EXPECT_TRUE(invokeParser("_ZGVqN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::Unknown); - - EXPECT_TRUE(invokeParser("_ZGVnN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); - - EXPECT_TRUE(invokeParser("_ZGVsN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::SVE); - - EXPECT_TRUE(invokeParser("_ZGVbN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::SSE); - - EXPECT_TRUE(invokeParser("_ZGVcN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::AVX); - - EXPECT_TRUE(invokeParser("_ZGVdN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::AVX2); - - EXPECT_TRUE(invokeParser("_ZGVeN2v_sin")); - EXPECT_EQ(ISA, VFISAKind::AVX512); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, LLVM_ISA) { - EXPECT_FALSE(invokeParser("_ZGV_LLVM_N2v_sin")); - EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_sin_(vector_name)", "vector_name")); + EXPECT_FALSE(invokeParser("_ZGV_LLVM_N2v_foo")); + EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::LLVM); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(Parameters.size(), (unsigned)1); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, InvalidMask) { - EXPECT_FALSE(invokeParser("_ZGVsK2v_sin")); + EXPECT_FALSE(invokeParser("_ZGVsK2v_foo")); } TEST_F(VFABIParserTest, InvalidParameter) { - EXPECT_FALSE(invokeParser("_ZGVsM2vX_sin")); + EXPECT_FALSE(invokeParser("_ZGVsM2vX_foo")); } TEST_F(VFABIParserTest, Align) { - EXPECT_TRUE(invokeParser("_ZGVsN2l2a2_sin")); + EXPECT_TRUE(invokeParser("_ZGVsN2l2a2_foo(vector_foo)", "void(i32)")); + EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; EXPECT_EQ(Parameters.size(), (unsigned)1); EXPECT_EQ(Parameters[0].Alignment, Align(2)); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); // Missing alignment value. - EXPECT_FALSE(invokeParser("_ZGVsM2l2a_sin")); + EXPECT_FALSE(invokeParser("_ZGVsM2l2a_foo")); // Invalid alignment token "x". - EXPECT_FALSE(invokeParser("_ZGVsM2l2ax_sin")); + EXPECT_FALSE(invokeParser("_ZGVsM2l2ax_foo")); // Alignment MUST be associated to a paramater. - EXPECT_FALSE(invokeParser("_ZGVsM2a2_sin")); + EXPECT_FALSE(invokeParser("_ZGVsM2a2_foo")); // Alignment must be a power of 2. - EXPECT_FALSE(invokeParser("_ZGVsN2l2a0_sin")); - EXPECT_TRUE(invokeParser("_ZGVsN2l2a1_sin")); - EXPECT_FALSE(invokeParser("_ZGVsN2l2a3_sin")); - EXPECT_FALSE(invokeParser("_ZGVsN2l2a6_sin")); + EXPECT_FALSE(invokeParser("_ZGVsN2l2a0_foo")); + EXPECT_TRUE(invokeParser("_ZGVsN2l2a1_foo")); + EXPECT_FALSE(invokeParser("_ZGVsN2l2a3_foo")); + EXPECT_FALSE(invokeParser("_ZGVsN2l2a6_foo")); } TEST_F(VFABIParserTest, ParseUniform) { - EXPECT_TRUE(invokeParser("_ZGVnN2u_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_FALSE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVnN2u_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)1); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::OMP_Uniform, 0})); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "_ZGVnN2u_sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); // Uniform doesn't expect extra data. - EXPECT_FALSE(invokeParser("_ZGVnN2u0_sin")); + EXPECT_FALSE(invokeParser("_ZGVnN2u0_foo")); } TEST_F(VFABIParserTest, ISAIndependentMangling) { // This test makes sure that the mangling of the parameters in // independent on the token. + const StringRef IRTy = + "void(i32, i32, i32, i32, ptr, i32, i32, i32, i32, i32)"; const SmallVector ExpectedParams = { VFParameter({0, VFParamKind::Vector, 0}), VFParameter({1, VFParamKind::OMP_LinearPos, 2}), @@ -351,61 +373,63 @@ TEST_F(VFABIParserTest, ISAIndependentMangling) { #define __COMMON_CHECKS \ do { \ EXPECT_EQ(VF, ElementCount::getFixed(2)); \ - EXPECT_FALSE(IsMasked()); \ + EXPECT_FALSE(isMasked()); \ + EXPECT_TRUE(matchParametersNum()) \ + << "Different number of scalar parameters"; \ EXPECT_EQ(Parameters.size(), (unsigned)10); \ EXPECT_EQ(Parameters, ExpectedParams); \ - EXPECT_EQ(ScalarName, "sin"); \ + EXPECT_EQ(ScalarName, "foo"); \ + EXPECT_EQ(VectorName, "vector_foo"); \ } while (0) // Advanced SIMD: = "n" - EXPECT_TRUE(invokeParser("_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVnN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); // SVE: = "s" - EXPECT_TRUE(invokeParser("_ZGVsN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVsN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::SVE); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVsN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); // SSE: = "b" - EXPECT_TRUE(invokeParser("_ZGVbN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVbN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::SSE); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVbN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); // AVX: = "c" - EXPECT_TRUE(invokeParser("_ZGVcN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVcN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::AVX); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVcN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); // AVX2: = "d" - EXPECT_TRUE(invokeParser("_ZGVdN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVdN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::AVX2); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVdN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); // AVX512: = "e" - EXPECT_TRUE(invokeParser("_ZGVeN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVeN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::AVX512); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVeN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); // LLVM: = "_LLVM_" internal vector function. EXPECT_TRUE(invokeParser( - "_ZGV_LLVM_N2vls2Ls27Us4Rs5l1L10U100R1000u_sin(vectorf)", "vectorf")); + "_ZGV_LLVM_N2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::LLVM); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "vectorf"); // Unknown ISA (randomly using "q"). This test will need update if // some targets decide to use "q" as their ISA token. - EXPECT_TRUE(invokeParser("_ZGVqN2vls2Ls27Us4Rs5l1L10U100R1000u_sin")); + EXPECT_TRUE(invokeParser( + "_ZGVqN2vls2Ls27Us4Rs5l1L10U100R1000u_foo(vector_foo)", IRTy)); EXPECT_EQ(ISA, VFISAKind::Unknown); __COMMON_CHECKS; - EXPECT_EQ(VectorName, "_ZGVqN2vls2Ls27Us4Rs5l1L10U100R1000u_sin"); #undef __COMMON_CHECKS } @@ -423,108 +447,167 @@ TEST_F(VFABIParserTest, MissingVectorNameTermination) { } TEST_F(VFABIParserTest, ParseMaskingNEON) { - EXPECT_TRUE(invokeParser("_ZGVnM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVnM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::AdvancedSIMD); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseMaskingSVE) { - EXPECT_TRUE(invokeParser("_ZGVsM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVsM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseMaskingSSE) { - EXPECT_TRUE(invokeParser("_ZGVbM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVbM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::SSE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseMaskingAVX) { - EXPECT_TRUE(invokeParser("_ZGVcM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVcM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::AVX); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseMaskingAVX2) { - EXPECT_TRUE(invokeParser("_ZGVdM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVdM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::AVX2); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseMaskingAVX512) { - EXPECT_TRUE(invokeParser("_ZGVeM2v_sin")); - EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_TRUE(invokeParser("_ZGVeM2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::AVX512); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(2)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseMaskingLLVM) { - EXPECT_TRUE(invokeParser("_ZGV_LLVM_M2v_sin(custom_vector_sin)", - "custom_vector_sin")); + EXPECT_TRUE(invokeParser("_ZGV_LLVM_M2v_foo(vector_foo)", "void(i32)")); + EXPECT_EQ(ISA, VFISAKind::LLVM); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; EXPECT_EQ(VF, ElementCount::getFixed(2)); - EXPECT_TRUE(IsMasked()); + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); +} + +TEST_F(VFABIParserTest, ParseScalableMaskingLLVM) { + EXPECT_FALSE(invokeParser("_ZGV_LLVM_Mxv_foo(vector_foo)")); +} + +TEST_F(VFABIParserTest, LLVM_InternalISA) { + EXPECT_FALSE(invokeParser("_ZGV_LLVM_N2v_foo")); + EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_foo(vector_foo)", "void(i32)")); EXPECT_EQ(ISA, VFISAKind::LLVM); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(Parameters.size(), (unsigned)1); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); +} + +TEST_F(VFABIParserTest, IntrinsicsInLLVMIsa) { + EXPECT_TRUE(invokeParser("_ZGV_LLVM_N4vv_llvm.pow.f32(__svml_powf4)", + "void(float, float)")); + EXPECT_EQ(ISA, VFISAKind::LLVM); + EXPECT_FALSE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getFixed(4)); + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::Vector})); + EXPECT_EQ(ScalarName, "llvm.pow.f32"); + EXPECT_EQ(VectorName, "__svml_powf4"); +} + +TEST_F(VFABIParserTest, ParseScalableRequiresDeclaration) { + const char *MangledName = "_ZGVsMxv_sin(custom_vg)"; + EXPECT_FALSE(invokeParser(MangledName)); + EXPECT_TRUE(invokeParser(MangledName, "void(i32)")); + EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "custom_vector_sin"); + EXPECT_EQ(VectorName, "custom_vg"); } -TEST_F(VFABIParserTest, ParseScalableMaskingLLVM) { - EXPECT_FALSE( - invokeParser("_ZGV_LLVM_Mxv_sin(custom_vector_sin)", "sin", "i32(i32)")); +TEST_F(VFABIParserTest, ZeroIsInvalidVLEN) { + EXPECT_FALSE(invokeParser("_ZGVeM0v_foo")); + EXPECT_FALSE(invokeParser("_ZGVeN0v_foo")); + EXPECT_FALSE(invokeParser("_ZGVsM0v_foo")); + EXPECT_FALSE(invokeParser("_ZGVsN0v_foo")); } TEST_F(VFABIParserTest, ParseScalableMaskingSVE) { - EXPECT_TRUE( - invokeParser("_ZGVsMxv_sin(custom_vector_sin)", "sin", "i32(i32)")); - EXPECT_TRUE(IsMasked()); - EXPECT_EQ(VF, ElementCount::getScalable(4)); + EXPECT_TRUE(invokeParser("_ZGVsMxv_foo(vector_foo)", "i32(i32)")); EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getScalable(4)); EXPECT_EQ(Parameters.size(), (unsigned)2); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); - EXPECT_EQ(ScalarName, "sin"); - EXPECT_EQ(VectorName, "custom_vector_sin"); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } TEST_F(VFABIParserTest, ParseScalableMaskingSVESincos) { EXPECT_TRUE(invokeParser("_ZGVsMxvl8l8_sincos(custom_vector_sincos)", - "sincos", "void(double, double *, double *)")); - EXPECT_EQ(VF, ElementCount::getScalable(2)); - EXPECT_TRUE(IsMasked()); + "void(double, ptr, ptr)")); EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(VF, ElementCount::getScalable(2)); EXPECT_EQ(Parameters.size(), (unsigned)4); EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::OMP_Linear, 8})); @@ -537,27 +620,42 @@ TEST_F(VFABIParserTest, ParseScalableMaskingSVESincos) { // Make sure that we get the correct VF if the return type is wider than any // parameter type. TEST_F(VFABIParserTest, ParseWiderReturnTypeSVE) { - EXPECT_TRUE( - invokeParser("_ZGVsMxvv_foo(vector_foo)", "foo", "i64(i32, i32)")); + EXPECT_TRUE(invokeParser("_ZGVsMxvv_foo(vector_foo)", "i64(i32, i32)")); + EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(Parameters.size(), (unsigned)3); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::Vector})); + EXPECT_EQ(Parameters[2], VFParameter({2, VFParamKind::GlobalPredicate})); EXPECT_EQ(VF, ElementCount::getScalable(2)); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } // Make sure we handle void return types. TEST_F(VFABIParserTest, ParseVoidReturnTypeSVE) { - EXPECT_TRUE(invokeParser("_ZGVsMxv_foo(vector_foo)", "foo", "void(i16)")); + EXPECT_TRUE(invokeParser("_ZGVsMxv_foo(vector_foo)", "void(i16)")); + EXPECT_EQ(ISA, VFISAKind::SVE); + EXPECT_TRUE(isMasked()); + EXPECT_TRUE(matchParametersNum()) << "Different number of scalar parameters"; + EXPECT_EQ(Parameters.size(), (unsigned)2); + EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); + EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::GlobalPredicate})); EXPECT_EQ(VF, ElementCount::getScalable(8)); + EXPECT_EQ(ScalarName, "foo"); + EXPECT_EQ(VectorName, "vector_foo"); } // Make sure we reject unsupported parameter types. TEST_F(VFABIParserTest, ParseUnsupportedElementTypeSVE) { - EXPECT_FALSE(invokeParser("_ZGVsMxv_foo(vector_foo)", "foo", "void(i128)")); + EXPECT_FALSE(invokeParser("_ZGVsMxv_foo(vector_foo)", "void(i128)")); } // Make sure we reject unsupported return types TEST_F(VFABIParserTest, ParseUnsupportedReturnTypeSVE) { - EXPECT_FALSE(invokeParser("_ZGVsMxv_foo(vector_foo)", "foo", "fp128(float)")); + EXPECT_FALSE(invokeParser("_ZGVsMxv_foo(vector_foo)", "fp128(float)")); } - class VFABIAttrTest : public testing::Test { protected: void SetUp() override { @@ -592,39 +690,6 @@ TEST_F(VFABIAttrTest, Read) { EXPECT_EQ(Mappings, Exp); } -TEST_F(VFABIParserTest, LLVM_InternalISA) { - EXPECT_FALSE(invokeParser("_ZGV_LLVM_N2v_sin")); - EXPECT_TRUE(invokeParser("_ZGV_LLVM_N2v_sin_(vector_name)", "vector_name")); - EXPECT_EQ(ISA, VFISAKind::LLVM); -} - -TEST_F(VFABIParserTest, IntrinsicsInLLVMIsa) { - EXPECT_TRUE(invokeParser("_ZGV_LLVM_N4vv_llvm.pow.f32(__svml_powf4)", - "__svml_powf4")); - EXPECT_EQ(VF, ElementCount::getFixed(4)); - EXPECT_FALSE(IsMasked()); - EXPECT_EQ(ISA, VFISAKind::LLVM); - EXPECT_EQ(Parameters.size(), (unsigned)2); - EXPECT_EQ(Parameters[0], VFParameter({0, VFParamKind::Vector})); - EXPECT_EQ(Parameters[1], VFParameter({1, VFParamKind::Vector})); - EXPECT_EQ(ScalarName, "llvm.pow.f32"); -} - -TEST_F(VFABIParserTest, ParseScalableRequiresDeclaration) { - const char *MangledName = "_ZGVsMxv_sin(custom_vg)"; - // The parser succeds only when the correct function definition of - // `custom_vg` is added to the module. - EXPECT_FALSE(invokeParser(MangledName)); - EXPECT_TRUE(invokeParser(MangledName, "sin", "double(double)")); -} - -TEST_F(VFABIParserTest, ZeroIsInvalidVLEN) { - EXPECT_FALSE(invokeParser("_ZGVeM0v_sin")); - EXPECT_FALSE(invokeParser("_ZGVeN0v_sin")); - EXPECT_FALSE(invokeParser("_ZGVsM0v_sin")); - EXPECT_FALSE(invokeParser("_ZGVsN0v_sin")); -} - static std::unique_ptr parseIR(LLVMContext &C, const char *IR) { SMDiagnostic Err; std::unique_ptr Mod = parseAssemblyString(IR, Err, C); @@ -642,9 +707,9 @@ entry: ret void } )IR"); - auto F = dyn_cast_or_null(M->getNamedValue("call")); + auto *F = dyn_cast_or_null(M->getNamedValue("call")); ASSERT_TRUE(F); - auto CI = dyn_cast(&F->front().front()); + auto *CI = dyn_cast(&F->front().front()); ASSERT_TRUE(CI); ASSERT_TRUE(CI->isIndirectCall()); auto Mappings = VFDatabase::getMappings(*CI); diff --git a/llvm/unittests/Analysis/VectorUtilsTest.cpp b/llvm/unittests/Analysis/VectorUtilsTest.cpp index 1b3a8b0259f0..14958aa646a0 100644 --- a/llvm/unittests/Analysis/VectorUtilsTest.cpp +++ b/llvm/unittests/Analysis/VectorUtilsTest.cpp @@ -17,7 +17,6 @@ #include "llvm/IR/NoFolder.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/SourceMgr.h" -#include "llvm/Support/KnownBits.h" #include "gtest/gtest.h" using namespace llvm; diff --git a/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp b/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp index 0b7f8f41bc53..5cb0310c0ad0 100644 --- a/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp +++ b/llvm/unittests/DebugInfo/DWARF/DWARFDebugInfoTest.cpp @@ -1615,6 +1615,42 @@ TEST(DWARFDebugInfo, TestFindRecurse) { EXPECT_EQ(AbsDieName, StringOpt.value_or(nullptr)); } +TEST(DWARFDebugInfo, TestSelfRecursiveType) { + typedef uint32_t AddrType; + Triple Triple = getDefaultTargetTripleForAddrSize(sizeof(AddrType)); + if (!isConfigurationSupported(Triple)) + GTEST_SKIP(); + + auto ExpectedDG = dwarfgen::Generator::create(Triple, 4); + ASSERT_THAT_EXPECTED(ExpectedDG, Succeeded()); + dwarfgen::Generator *DG = ExpectedDG.get().get(); + dwarfgen::CompileUnit &CU = DG->addCompileUnit(); + dwarfgen::DIE CUDie = CU.getUnitDIE(); + + // Create an invalid self-recursive typedef. + dwarfgen::DIE TypedefDie = CUDie.addChild(DW_TAG_typedef); + TypedefDie.addAttribute(DW_AT_name, DW_FORM_strp, "illegal"); + TypedefDie.addAttribute(DW_AT_type, DW_FORM_ref_addr, TypedefDie); + + MemoryBufferRef FileBuffer(DG->generate(), "dwarf"); + auto Obj = object::ObjectFile::createObjectFile(FileBuffer); + EXPECT_TRUE((bool)Obj); + std::unique_ptr DwarfContext = DWARFContext::create(**Obj); + + // Verify the number of compile units is correct. + uint32_t NumCUs = DwarfContext->getNumCompileUnits(); + EXPECT_EQ(NumCUs, 1u); + DWARFCompileUnit *U = cast(DwarfContext->getUnitAtIndex(0)); + { + DWARFDie CUDie = U->getUnitDIE(false); + EXPECT_TRUE(CUDie.isValid()); + DWARFDie TypedefDie = CUDie.getFirstChild(); + + // Test that getTypeSize doesn't get into an infinite loop. + EXPECT_EQ(TypedefDie.getTypeSize(sizeof(AddrType)), std::nullopt); + } +} + TEST(DWARFDebugInfo, TestDwarfToFunctions) { // Test all of the dwarf::toXXX functions that take a // std::optional and extract the values from it. diff --git a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt index 37768e91fd44..f102ba59e375 100644 --- a/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/unittests/ExecutionEngine/Orc/CMakeLists.txt @@ -5,6 +5,7 @@ set(LLVM_LINK_COMPONENTS IRReader JITLink Object + OrcDebugging OrcJIT OrcShared OrcTargetProcess diff --git a/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp b/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp index cbdd4af47e1d..65d2f57234d0 100644 --- a/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/OrcCAPITest.cpp @@ -9,10 +9,12 @@ #include "llvm-c/Core.h" #include "llvm-c/Error.h" #include "llvm-c/LLJIT.h" +#include "llvm-c/LLJITUtils.h" #include "llvm-c/Orc.h" #include "gtest/gtest.h" #include "llvm/ExecutionEngine/Orc/CompileUtils.h" +#include "llvm/ExecutionEngine/Orc/TargetProcess/JITLoaderGDB.h" #include "llvm/IR/LLVMContext.h" #include "llvm/IR/Module.h" #include "llvm/IRReader/IRReader.h" @@ -211,6 +213,20 @@ constexpr StringRef SumExample = } )"; +constexpr StringRef SumDebugExample = + R"( + define i32 @sum(i32 %x, i32 %y) { + entry: + %r = add nsw i32 %x, %y + ret i32 %r + } + !llvm.module.flags = !{!0} + !llvm.dbg.cu = !{!1} + !0 = !{i32 2, !"Debug Info Version", i32 3} + !1 = distinct !DICompileUnit(language: DW_LANG_C99, file: !2, emissionKind: FullDebug) + !2 = !DIFile(filename: "sum.c", directory: "/tmp") + )"; + } // end anonymous namespace. // Consumes the given error ref and returns the string error message. @@ -494,6 +510,75 @@ TEST_F(OrcCAPITestBase, AddObjectBuffer) { ASSERT_TRUE(!!SumAddr); } +// This must be kept in sync with gdb/gdb/jit.h . +extern "C" { + +typedef enum { + JIT_NOACTION = 0, + JIT_REGISTER_FN, + JIT_UNREGISTER_FN +} jit_actions_t; + +struct jit_code_entry { + struct jit_code_entry *next_entry; + struct jit_code_entry *prev_entry; + const char *symfile_addr; + uint64_t symfile_size; +}; + +struct jit_descriptor { + uint32_t version; + // This should be jit_actions_t, but we want to be specific about the + // bit-width. + uint32_t action_flag; + struct jit_code_entry *relevant_entry; + struct jit_code_entry *first_entry; +}; + +// We put information about the JITed function in this global, which the +// debugger reads. Make sure to specify the version statically, because the +// debugger checks the version before we can set it during runtime. +extern struct jit_descriptor __jit_debug_descriptor; + +static void *findLastDebugDescriptorEntryPtr() { + struct jit_code_entry *Last = __jit_debug_descriptor.first_entry; + while (Last && Last->next_entry) + Last = Last->next_entry; + return Last; +} +} + +#if defined(_AIX) or not(defined(__ELF__) or defined(__MACH__)) +TEST_F(OrcCAPITestBase, DISABLED_EnableDebugSupport) { +#else +static LLVM_ATTRIBUTE_USED void linkComponents() { + errs() << "Linking in runtime functions\n" + << (void *)&llvm_orc_registerJITLoaderGDBWrapper << '\n' + << (void *)&llvm_orc_registerJITLoaderGDBAllocAction << '\n'; +} +TEST_F(OrcCAPITestBase, EnableDebugSupport) { +#endif + if (LLVMErrorRef E = LLVMOrcLLJITEnableDebugSupport(Jit)) + FAIL() << "Error testing LLJIT debug support (triple = " << TargetTriple + << "): " << toString(E); + + void *Before = findLastDebugDescriptorEntryPtr(); + LLVMMemoryBufferRef ObjBuffer = createTestObject(SumDebugExample, "sum.ll"); + LLVMOrcObjectLayerRef ObjLayer = LLVMOrcLLJITGetObjLinkingLayer(Jit); + if (LLVMErrorRef E = + LLVMOrcObjectLayerAddObjectFile(ObjLayer, MainDylib, ObjBuffer)) + FAIL() << "Failed to add object file to ObjLinkingLayer (triple = " + << TargetTriple << "): " << toString(E); + + LLVMOrcJITTargetAddress SumAddr; + if (LLVMErrorRef E = LLVMOrcLLJITLookup(Jit, &SumAddr, "sum")) + FAIL() << "Symbol \"sum\" was not added into JIT (triple = " << TargetTriple + << "): " << toString(E); + + void *After = findLastDebugDescriptorEntryPtr(); + ASSERT_NE(Before, After); +} + #if defined(_AIX) TEST_F(OrcCAPITestBase, DISABLED_ExecutionTest) { #else diff --git a/llvm/unittests/Support/Chrono.cpp b/llvm/unittests/Support/Chrono.cpp index daf8a8a350f0..7dfc5dd2c293 100644 --- a/llvm/unittests/Support/Chrono.cpp +++ b/llvm/unittests/Support/Chrono.cpp @@ -7,7 +7,6 @@ //===----------------------------------------------------------------------===// #include "llvm/Support/Chrono.h" -#include "llvm/ADT/SmallVector.h" #include "llvm/Support/FormatVariadic.h" #include "gtest/gtest.h" diff --git a/llvm/unittests/Support/CommandLineTest.cpp b/llvm/unittests/Support/CommandLineTest.cpp index 381fe70b6b48..762ac0ea9c36 100644 --- a/llvm/unittests/Support/CommandLineTest.cpp +++ b/llvm/unittests/Support/CommandLineTest.cpp @@ -28,7 +28,6 @@ #include #include #include -#include using namespace llvm; using llvm::unittest::TempDir; @@ -1347,29 +1346,32 @@ struct AutoDeleteFile { } }; +static std::string interceptStdout(std::function F) { + outs().flush(); // flush any output from previous tests + AutoDeleteFile File; + { + OutputRedirector Stdout(fileno(stdout)); + if (!Stdout.Valid) + return ""; + File.FilePath = Stdout.FilePath; + F(); + outs().flush(); + } + auto Buffer = MemoryBuffer::getFile(File.FilePath); + if (!Buffer) + return ""; + return Buffer->get()->getBuffer().str(); +} + template class PrintOptionTestBase : public ::testing::Test { public: // Return std::string because the output of a failing EXPECT check is // unreadable for StringRef. It also avoids any lifetime issues. template std::string runTest(Ts... OptionAttributes) { - outs().flush(); // flush any output from previous tests - AutoDeleteFile File; - { - OutputRedirector Stdout(fileno(stdout)); - if (!Stdout.Valid) - return ""; - File.FilePath = Stdout.FilePath; - - StackOption TestOption(Opt, cl::desc(HelpText), - OptionAttributes...); - Func(TestOption); - outs().flush(); - } - auto Buffer = MemoryBuffer::getFile(File.FilePath); - if (!Buffer) - return ""; - return Buffer->get()->getBuffer().str(); + StackOption TestOption(Opt, cl::desc(HelpText), + OptionAttributes...); + return interceptStdout([&]() { Func(TestOption); }); } enum class OptionValue { Val }; @@ -2206,4 +2208,40 @@ TEST(CommandLineTest, DefaultValue) { EXPECT_EQ(1, StrInitOption.getNumOccurrences()); } +TEST(CommandLineTest, HelpWithoutSubcommands) { + // Check that the help message does not contain the "[subcommand]" placeholder + // and the "SUBCOMMANDS" section if there are no subcommands. + cl::ResetCommandLineParser(); + StackOption Opt("opt", cl::init(false)); + const char *args[] = {"prog"}; + EXPECT_TRUE(cl::ParseCommandLineOptions(std::size(args), args, StringRef(), + &llvm::nulls())); + auto Output = interceptStdout([]() { cl::PrintHelpMessage(); }); + EXPECT_NE(std::string::npos, Output.find("USAGE: prog [options]")) << Output; + EXPECT_EQ(std::string::npos, Output.find("SUBCOMMANDS:")) << Output; + cl::ResetCommandLineParser(); +} + +TEST(CommandLineTest, HelpWithSubcommands) { + // Check that the help message contains the "[subcommand]" placeholder in the + // "USAGE" line and describes subcommands. + cl::ResetCommandLineParser(); + StackSubCommand SC1("sc1", "First Subcommand"); + StackSubCommand SC2("sc2", "Second Subcommand"); + StackOption SC1Opt("sc1", cl::sub(SC1), cl::init(false)); + StackOption SC2Opt("sc2", cl::sub(SC2), cl::init(false)); + const char *args[] = {"prog"}; + EXPECT_TRUE(cl::ParseCommandLineOptions(std::size(args), args, StringRef(), + &llvm::nulls())); + auto Output = interceptStdout([]() { cl::PrintHelpMessage(); }); + EXPECT_NE(std::string::npos, + Output.find("USAGE: prog [subcommand] [options]")) + << Output; + EXPECT_NE(std::string::npos, Output.find("SUBCOMMANDS:")) << Output; + EXPECT_NE(std::string::npos, Output.find("sc1 - First Subcommand")) << Output; + EXPECT_NE(std::string::npos, Output.find("sc2 - Second Subcommand")) + << Output; + cl::ResetCommandLineParser(); +} + } // anonymous namespace diff --git a/llvm/unittests/Support/InstructionCostTest.cpp b/llvm/unittests/Support/InstructionCostTest.cpp index 2f634110bd51..4e2842d8ada9 100644 --- a/llvm/unittests/Support/InstructionCostTest.cpp +++ b/llvm/unittests/Support/InstructionCostTest.cpp @@ -8,7 +8,6 @@ #include "llvm/Support/InstructionCost.h" #include "gtest/gtest.h" -#include using namespace llvm; diff --git a/llvm/unittests/Support/MemoryTest.cpp b/llvm/unittests/Support/MemoryTest.cpp index 019ece9c1725..0164492829cc 100644 --- a/llvm/unittests/Support/MemoryTest.cpp +++ b/llvm/unittests/Support/MemoryTest.cpp @@ -9,7 +9,6 @@ #include "llvm/Support/Memory.h" #include "llvm/Support/Process.h" #include "gtest/gtest.h" -#include #include #if defined(__NetBSD__) diff --git a/llvm/unittests/Support/Path.cpp b/llvm/unittests/Support/Path.cpp index 35a01aa27667..a7b7e6a0f504 100644 --- a/llvm/unittests/Support/Path.cpp +++ b/llvm/unittests/Support/Path.cpp @@ -882,7 +882,7 @@ TEST_F(FileSystemTest, TempFiles) { int FD2; SmallString<64> TempPath2; ASSERT_NO_ERROR(fs::createTemporaryFile("prefix", "temp", FD2, TempPath2)); - ASSERT_TRUE(TempPath2.endswith(".temp")); + ASSERT_TRUE(TempPath2.ends_with(".temp")); ASSERT_NE(TempPath.str(), TempPath2.str()); fs::file_status A, B; @@ -908,7 +908,7 @@ TEST_F(FileSystemTest, TempFiles) { SmallString<64> TempPath3; ASSERT_NO_ERROR(fs::createTemporaryFile("prefix", "", TempPath3)); - ASSERT_FALSE(TempPath3.endswith(".")); + ASSERT_FALSE(TempPath3.ends_with(".")); FileRemover Cleanup3(TempPath3); // Create a hard link to Temp1. @@ -1515,13 +1515,13 @@ TEST(Support, NormalizePath) { const char *Path7a = "~/aaa"; SmallString<64> Path7(Path7a); path::native(Path7, path::Style::windows_backslash); - EXPECT_TRUE(Path7.endswith("\\aaa")); - EXPECT_TRUE(Path7.startswith(PathHome)); + EXPECT_TRUE(Path7.ends_with("\\aaa")); + EXPECT_TRUE(Path7.starts_with(PathHome)); EXPECT_EQ(Path7.size(), PathHome.size() + strlen(Path7a + 1)); Path7 = Path7a; path::native(Path7, path::Style::windows_slash); - EXPECT_TRUE(Path7.endswith("/aaa")); - EXPECT_TRUE(Path7.startswith(PathHome)); + EXPECT_TRUE(Path7.ends_with("/aaa")); + EXPECT_TRUE(Path7.starts_with(PathHome)); EXPECT_EQ(Path7.size(), PathHome.size() + strlen(Path7a + 1)); const char *Path8a = "~"; diff --git a/llvm/unittests/TableGen/AutomataTest.cpp b/llvm/unittests/TableGen/AutomataTest.cpp index 53f66127292f..833cf51b19a1 100644 --- a/llvm/unittests/TableGen/AutomataTest.cpp +++ b/llvm/unittests/TableGen/AutomataTest.cpp @@ -6,7 +6,6 @@ // //===----------------------------------------------------------------------===// -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/Support/Debug.h" #include "llvm/Support/Automaton.h" diff --git a/llvm/unittests/TargetParser/TargetParserTest.cpp b/llvm/unittests/TargetParser/TargetParserTest.cpp index abd87b455ace..687b56cc5aa2 100644 --- a/llvm/unittests/TargetParser/TargetParserTest.cpp +++ b/llvm/unittests/TargetParser/TargetParserTest.cpp @@ -416,13 +416,13 @@ INSTANTIATE_TEST_SUITE_P( ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_FP16 | ARM::AEK_RAS | ARM::AEK_DOTPROD, "8.2-A"), - ARMCPUTestParams("neoverse-n2", "armv8.5-a", "crypto-neon-fp-armv8", + ARMCPUTestParams("neoverse-n2", "armv9-a", "neon-fp-armv8", ARM::AEK_CRC | ARM::AEK_HWDIVTHUMB | ARM::AEK_HWDIVARM | ARM::AEK_MP | ARM::AEK_SEC | ARM::AEK_VIRT | ARM::AEK_DSP | ARM::AEK_BF16 | ARM::AEK_DOTPROD | ARM::AEK_RAS | ARM::AEK_I8MM | ARM::AEK_SB, - "8.5-A"), + "9-A"), ARMCPUTestParams("neoverse-v1", "armv8.4-a", "crypto-neon-fp-armv8", ARM::AEK_SEC | ARM::AEK_MP | ARM::AEK_VIRT | ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | @@ -469,13 +469,19 @@ INSTANTIATE_TEST_SUITE_P( ARM::AEK_FP | ARM::AEK_RAS | ARM::AEK_LOB | ARM::AEK_FP16 | ARM::AEK_PACBTI, "8.1-M.Mainline"), + ARMCPUTestParams("cortex-m52", "armv8.1-m.main", + "fp-armv8-fullfp16-d16", + ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP | ARM::AEK_SIMD | + ARM::AEK_FP | ARM::AEK_RAS | ARM::AEK_LOB | + ARM::AEK_FP16 | ARM::AEK_PACBTI, + "8.1-M.Mainline"), ARMCPUTestParams("iwmmxt", "iwmmxt", "none", ARM::AEK_NONE, "iwmmxt"), ARMCPUTestParams("xscale", "xscale", "none", ARM::AEK_NONE, "xscale"), ARMCPUTestParams("swift", "armv7s", "neon-vfpv4", ARM::AEK_HWDIVARM | ARM::AEK_HWDIVTHUMB | ARM::AEK_DSP, "7-S"))); -static constexpr unsigned NumARMCPUArchs = 89; +static constexpr unsigned NumARMCPUArchs = 90; TEST(TargetParserTest, testARMCPUArchList) { SmallVector List; @@ -1515,11 +1521,9 @@ INSTANTIATE_TEST_SUITE_P( AArch64::AEK_SSBS})), "8.2-A"), ARMCPUTestParams( - "neoverse-n2", "armv8.5-a", "crypto-neon-fp-armv8", + "neoverse-n2", "armv9-a", "crypto-neon-fp-armv8", (AArch64::ExtensionBitset( - {AArch64::AEK_CRC, AArch64::AEK_AES, - AArch64::AEK_SHA2, AArch64::AEK_SHA3, - AArch64::AEK_SM4, AArch64::AEK_FP, + {AArch64::AEK_CRC, AArch64::AEK_FP, AArch64::AEK_SIMD, AArch64::AEK_FP16, AArch64::AEK_RAS, AArch64::AEK_LSE, AArch64::AEK_SVE, AArch64::AEK_DOTPROD, diff --git a/llvm/unittests/TextAPI/RecordTests.cpp b/llvm/unittests/TextAPI/RecordTests.cpp index 076137de6ff1..dbe58ca0eea5 100644 --- a/llvm/unittests/TextAPI/RecordTests.cpp +++ b/llvm/unittests/TextAPI/RecordTests.cpp @@ -48,7 +48,7 @@ TEST(TAPIRecord, SimpleObjC) { TEST(TAPIRecord, SimpleSlice) { Triple T("arm64-apple-macosx13.3"); RecordsSlice Slice(T); - EXPECT_TRUE(Slice.isEmpty()); + EXPECT_TRUE(Slice.empty()); Slice.addRecord("_OBJC_CLASS_$_NSObject", SymbolFlags::None, GlobalRecord::Kind::Unknown, RecordLinkage::Rexported); Slice.addRecord("_OBJC_METACLASS_$_NSObject", SymbolFlags::None, @@ -59,7 +59,7 @@ TEST(TAPIRecord, SimpleSlice) { GlobalRecord::Kind::Unknown, RecordLinkage::Exported); Slice.addRecord("_foo", SymbolFlags::WeakDefined | SymbolFlags::Rexported, GlobalRecord::Kind::Variable, RecordLinkage::Rexported); - EXPECT_FALSE(Slice.isEmpty()); + EXPECT_FALSE(Slice.empty()); // Check global. EXPECT_FALSE(Slice.findGlobal("_foo", GlobalRecord::Kind::Function)); @@ -108,7 +108,7 @@ TEST(TAPIRecord, SimpleSlice) { TEST(TAPIRecord, LibraryAttrs) { Triple T("arm64-apple-ios15.1"); RecordsSlice Slice(T); - EXPECT_TRUE(Slice.isEmpty()); + EXPECT_TRUE(Slice.empty()); auto BA = Slice.getBinaryAttrs(); EXPECT_TRUE(Slice.hasBinaryAttrs()); diff --git a/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp b/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp index 2bab3aa15baa..1638ede7f3d4 100644 --- a/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp +++ b/llvm/unittests/tools/llvm-exegesis/X86/SnippetFileTest.cpp @@ -209,6 +209,16 @@ TEST_F(X86SnippetFileTest, MemoryMappingNoDefinition) { consumeError(std::move(Error)); } +TEST_F(X86SnippetFileTest, SnippetAddress) { + auto Snippets = TestCommon(R"( + # LLVM-EXEGESIS-SNIPPET-ADDRESS 10000 + )"); + ASSERT_TRUE(static_cast(Snippets)); + EXPECT_THAT(*Snippets, SizeIs(1)); + const auto &Snippet = (*Snippets)[0]; + EXPECT_EQ(Snippet.Key.SnippetAddress, 0x10000); +} + } // namespace } // namespace exegesis } // namespace llvm diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index 8d9ded1b2ac5..f1b2ff68e343 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -282,6 +282,10 @@ static std::string getScopedName(unsigned Scope, const std::string &Name) { return ("pred:" + Twine(Scope) + ":" + Name).str(); } +static std::string getMangledRootDefName(StringRef DefOperandName) { + return ("DstI[" + DefOperandName + "]").str(); +} + //===- GlobalISelEmitter class --------------------------------------------===// static Expected getInstResultType(const TreePatternNode *Dst) { @@ -1499,8 +1503,13 @@ Expected GlobalISelEmitter::importExplicitDefRenderers( if (DstNumDefs == 0) return InsertPt; - for (unsigned I = 0; I < SrcNumDefs; ++I) - DstMIBuilder.addRenderer(DstI->Operands[I].Name); + for (unsigned I = 0; I < SrcNumDefs; ++I) { + std::string OpName = getMangledRootDefName(DstI->Operands[I].Name); + // CopyRenderer saves a StringRef, so cannot pass OpName itself - + // let's use a string with an appropriate lifetime. + StringRef PermanentRef = M.getOperandMatcher(OpName).getSymbolicName(); + DstMIBuilder.addRenderer(PermanentRef); + } // Some instructions have multiple defs, but are missing a type entry // (e.g. s_cc_out operands). @@ -2013,16 +2022,17 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { const TypeSetByHwMode &VTy = Src->getExtType(I); const auto &DstIOperand = DstI.Operands[OpIdx]; - Record *DstIOpRec = DstIOperand.Rec; + PointerUnion MatchedRC = + DstIOperand.Rec; if (DstIName == "COPY_TO_REGCLASS") { - DstIOpRec = getInitValueAsRegClass(Dst->getChild(1)->getLeafValue()); + MatchedRC = getInitValueAsRegClass(Dst->getChild(1)->getLeafValue()); - if (DstIOpRec == nullptr) + if (MatchedRC.isNull()) return failedImport( "COPY_TO_REGCLASS operand #1 isn't a register class"); } else if (DstIName == "REG_SEQUENCE") { - DstIOpRec = getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()); - if (DstIOpRec == nullptr) + MatchedRC = getInitValueAsRegClass(Dst->getChild(0)->getLeafValue()); + if (MatchedRC.isNull()) return failedImport("REG_SEQUENCE operand #0 isn't a register class"); } else if (DstIName == "EXTRACT_SUBREG") { auto InferredClass = inferRegClassFromPattern(Dst->getChild(0)); @@ -2032,7 +2042,7 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { // We can assume that a subregister is in the same bank as it's super // register. - DstIOpRec = (*InferredClass)->getDef(); + MatchedRC = (*InferredClass)->getDef(); } else if (DstIName == "INSERT_SUBREG") { auto MaybeSuperClass = inferSuperRegisterClassForNode( VTy, Dst->getChild(0), Dst->getChild(2)); @@ -2042,34 +2052,30 @@ Expected GlobalISelEmitter::runOnPattern(const PatternToMatch &P) { // Move to the next pattern here, because the register class we found // doesn't necessarily have a record associated with it. So, we can't // set DstIOpRec using this. - OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); - OM.setSymbolicName(DstIOperand.Name); - M.defineOperand(OM.getSymbolicName(), OM); - OM.addPredicate(**MaybeSuperClass); - ++OpIdx; - continue; + MatchedRC = *MaybeSuperClass; } else if (DstIName == "SUBREG_TO_REG") { auto MaybeRegClass = inferSuperRegisterClass(VTy, Dst->getChild(2)); if (!MaybeRegClass) return failedImport( "Cannot infer register class for SUBREG_TO_REG operand #0"); - OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); - OM.setSymbolicName(DstIOperand.Name); - M.defineOperand(OM.getSymbolicName(), OM); - OM.addPredicate(**MaybeRegClass); - ++OpIdx; - continue; - } else if (DstIOpRec->isSubClassOf("RegisterOperand")) - DstIOpRec = DstIOpRec->getValueAsDef("RegClass"); - else if (!DstIOpRec->isSubClassOf("RegisterClass")) + MatchedRC = *MaybeRegClass; + } else if (MatchedRC.get()->isSubClassOf("RegisterOperand")) + MatchedRC = MatchedRC.get()->getValueAsDef("RegClass"); + else if (!MatchedRC.get()->isSubClassOf("RegisterClass")) return failedImport("Dst MI def isn't a register class" + to_string(*Dst)); OperandMatcher &OM = InsnMatcher.getOperand(OpIdx); - OM.setSymbolicName(DstIOperand.Name); + // The operand names declared in the DstI instruction are unrelated to + // those used in pattern's source and destination DAGs, so mangle the + // former to prevent implicitly adding unexpected + // GIM_CheckIsSameOperand predicates by the defineOperand method. + OM.setSymbolicName(getMangledRootDefName(DstIOperand.Name)); M.defineOperand(OM.getSymbolicName(), OM); + if (MatchedRC.is()) + MatchedRC = &Target.getRegisterClass(MatchedRC.get()); OM.addPredicate( - Target.getRegisterClass(DstIOpRec)); + *MatchedRC.get()); ++OpIdx; } diff --git a/llvm/utils/TableGen/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/GlobalISelMatchTable.cpp index 5a4d32a34e2b..481f3f16e013 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTable.cpp +++ b/llvm/utils/TableGen/GlobalISelMatchTable.cpp @@ -243,7 +243,7 @@ void MatchTable::emitDeclaration(raw_ostream &OS) const { if (I->Flags & MatchTableRecord::MTRF_Outdent) Indentation -= 2; } - OS << "};\n"; + OS << "}; // Size: " << (CurrentSize * 8) << " bytes\n"; } MatchTable MatchTable::buildTable(ArrayRef Rules, bool WithCoverage, diff --git a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h index c30198f11195..7e952d6df309 100644 --- a/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h +++ b/llvm/utils/TableGen/GlobalISelMatchTableExecutorEmitter.h @@ -20,7 +20,6 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include -#include namespace llvm { class CodeGenTarget; diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp index 3402809c4828..28604c5600bf 100644 --- a/llvm/utils/TableGen/IntrinsicEmitter.cpp +++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp @@ -12,7 +12,6 @@ #include "CodeGenIntrinsics.h" #include "SequenceToOffsetTable.h" -#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/StringExtras.h" diff --git a/llvm/utils/TableGen/X86DisassemblerTables.cpp b/llvm/utils/TableGen/X86DisassemblerTables.cpp index f879a9f5e409..959e0fda50b8 100644 --- a/llvm/utils/TableGen/X86DisassemblerTables.cpp +++ b/llvm/utils/TableGen/X86DisassemblerTables.cpp @@ -31,39 +31,47 @@ using namespace X86Disassembler; /// @param insnContext - The instruction class to transform to a string. /// @return - A statically-allocated string constant that contains the /// name of the instruction class. -static inline const char* stringForContext(InstructionContext insnContext) { +static inline const char *stringForContext(InstructionContext insnContext) { switch (insnContext) { default: llvm_unreachable("Unhandled instruction class"); -#define ENUM_ENTRY(n, r, d) case n: return #n; break; -#define ENUM_ENTRY_K_B(n, r, d) ENUM_ENTRY(n, r, d) ENUM_ENTRY(n##_K_B, r, d)\ - ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) ENUM_ENTRY(n##_B, r, d)\ - ENUM_ENTRY(n##_KZ_B, r, d) - INSTRUCTION_CONTEXTS +#define ENUM_ENTRY(n, r, d) \ + case n: \ + return #n; \ + break; +#define ENUM_ENTRY_K_B(n, r, d) \ + ENUM_ENTRY(n, r, d) \ + ENUM_ENTRY(n##_K_B, r, d) ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) \ + ENUM_ENTRY(n##_B, r, d) ENUM_ENTRY(n##_KZ_B, r, d) + INSTRUCTION_CONTEXTS #undef ENUM_ENTRY #undef ENUM_ENTRY_K_B } } /// stringForOperandType - Like stringForContext, but for OperandTypes. -static inline const char* stringForOperandType(OperandType type) { +static inline const char *stringForOperandType(OperandType type) { switch (type) { default: llvm_unreachable("Unhandled type"); -#define ENUM_ENTRY(i, d) case i: return #i; - TYPES +#define ENUM_ENTRY(i, d) \ + case i: \ + return #i; + TYPES #undef ENUM_ENTRY } } /// stringForOperandEncoding - like stringForContext, but for /// OperandEncodings. -static inline const char* stringForOperandEncoding(OperandEncoding encoding) { +static inline const char *stringForOperandEncoding(OperandEncoding encoding) { switch (encoding) { default: llvm_unreachable("Unhandled encoding"); -#define ENUM_ENTRY(i, d) case i: return #i; - ENCODINGS +#define ENUM_ENTRY(i, d) \ + case i: \ + return #i; + ENCODINGS #undef ENUM_ENTRY } } @@ -83,17 +91,17 @@ static inline bool inheritsFrom(InstructionContext child, switch (parent) { case IC: - return(inheritsFrom(child, IC_64BIT, AdSize64) || - (noPrefix && inheritsFrom(child, IC_OPSIZE, noPrefix)) || - inheritsFrom(child, IC_ADSIZE) || - (noPrefix && inheritsFrom(child, IC_XD, noPrefix)) || - (noPrefix && inheritsFrom(child, IC_XS, noPrefix))); + return (inheritsFrom(child, IC_64BIT, AdSize64) || + (noPrefix && inheritsFrom(child, IC_OPSIZE, noPrefix)) || + inheritsFrom(child, IC_ADSIZE) || + (noPrefix && inheritsFrom(child, IC_XD, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_XS, noPrefix))); case IC_64BIT: - return(inheritsFrom(child, IC_64BIT_REXW) || - (noPrefix && inheritsFrom(child, IC_64BIT_OPSIZE, noPrefix)) || - (!AdSize64 && inheritsFrom(child, IC_64BIT_ADSIZE)) || - (noPrefix && inheritsFrom(child, IC_64BIT_XD, noPrefix)) || - (noPrefix && inheritsFrom(child, IC_64BIT_XS, noPrefix))); + return (inheritsFrom(child, IC_64BIT_REXW) || + (noPrefix && inheritsFrom(child, IC_64BIT_OPSIZE, noPrefix)) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_ADSIZE)) || + (noPrefix && inheritsFrom(child, IC_64BIT_XD, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_64BIT_XS, noPrefix))); case IC_OPSIZE: return inheritsFrom(child, IC_64BIT_OPSIZE) || inheritsFrom(child, IC_OPSIZE_ADSIZE); @@ -118,20 +126,20 @@ static inline bool inheritsFrom(InstructionContext child, case IC_XS_ADSIZE: return inheritsFrom(child, IC_64BIT_XS_ADSIZE); case IC_64BIT_REXW: - return((noPrefix && inheritsFrom(child, IC_64BIT_REXW_XS, noPrefix)) || - (noPrefix && inheritsFrom(child, IC_64BIT_REXW_XD, noPrefix)) || - (noPrefix && inheritsFrom(child, IC_64BIT_REXW_OPSIZE, noPrefix)) || - (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE))); + return ((noPrefix && inheritsFrom(child, IC_64BIT_REXW_XS, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_64BIT_REXW_XD, noPrefix)) || + (noPrefix && inheritsFrom(child, IC_64BIT_REXW_OPSIZE, noPrefix)) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE))); case IC_64BIT_OPSIZE: return inheritsFrom(child, IC_64BIT_REXW_OPSIZE) || (!AdSize64 && inheritsFrom(child, IC_64BIT_OPSIZE_ADSIZE)) || (!AdSize64 && inheritsFrom(child, IC_64BIT_REXW_ADSIZE)); case IC_64BIT_XD: - return(inheritsFrom(child, IC_64BIT_REXW_XD) || - (!AdSize64 && inheritsFrom(child, IC_64BIT_XD_ADSIZE))); + return (inheritsFrom(child, IC_64BIT_REXW_XD) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_XD_ADSIZE))); case IC_64BIT_XS: - return(inheritsFrom(child, IC_64BIT_REXW_XS) || - (!AdSize64 && inheritsFrom(child, IC_64BIT_XS_ADSIZE))); + return (inheritsFrom(child, IC_64BIT_REXW_XS) || + (!AdSize64 && inheritsFrom(child, IC_64BIT_XS_ADSIZE))); case IC_64BIT_XD_OPSIZE: case IC_64BIT_XS_OPSIZE: return false; @@ -410,10 +418,8 @@ static inline bool inheritsFrom(InstructionContext child, (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_K_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_K_B)); case IC_EVEX_OPSIZE_K_B: - return (VEX_LIG && WIG && - inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K_B)) || - (VEX_LIG && WIG && - inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K_B)) || + return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_K_B)) || + (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_K_B)) || (WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_K_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_K_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_K_B)); @@ -436,10 +442,8 @@ static inline bool inheritsFrom(InstructionContext child, (VEX_LIG && inheritsFrom(child, IC_EVEX_L_XD_KZ_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_XD_KZ_B)); case IC_EVEX_OPSIZE_KZ_B: - return (VEX_LIG && WIG && - inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B)) || - (VEX_LIG && WIG && - inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B)) || + return (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L_W_OPSIZE_KZ_B)) || + (VEX_LIG && WIG && inheritsFrom(child, IC_EVEX_L2_W_OPSIZE_KZ_B)) || (WIG && inheritsFrom(child, IC_EVEX_W_OPSIZE_KZ_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L_OPSIZE_KZ_B)) || (VEX_LIG && inheritsFrom(child, IC_EVEX_L2_OPSIZE_KZ_B)); @@ -558,8 +562,8 @@ static inline bool inheritsFrom(InstructionContext child, case IC_EVEX_L2_W_OPSIZE_KZ_B: return false; default: - errs() << "Unknown instruction class: " << - stringForContext((InstructionContext)parent) << "\n"; + errs() << "Unknown instruction class: " + << stringForContext((InstructionContext)parent) << "\n"; llvm_unreachable("Unknown instruction class"); } } @@ -577,12 +581,12 @@ static inline bool outranks(InstructionContext upper, assert(lower < IC_max); #define ENUM_ENTRY(n, r, d) r, -#define ENUM_ENTRY_K_B(n, r, d) ENUM_ENTRY(n, r, d) \ - ENUM_ENTRY(n##_K_B, r, d) ENUM_ENTRY(n##_KZ_B, r, d) \ - ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) ENUM_ENTRY(n##_B, r, d) - static int ranks[IC_max] = { - INSTRUCTION_CONTEXTS - }; +#define ENUM_ENTRY_K_B(n, r, d) \ + ENUM_ENTRY(n, r, d) \ + ENUM_ENTRY(n##_K_B, r, d) \ + ENUM_ENTRY(n##_KZ_B, r, d) ENUM_ENTRY(n##_KZ, r, d) ENUM_ENTRY(n##_K, r, d) \ + ENUM_ENTRY(n##_B, r, d) + static int ranks[IC_max] = {INSTRUCTION_CONTEXTS}; #undef ENUM_ENTRY #undef ENUM_ENTRY_K_B @@ -605,19 +609,19 @@ static ModRMDecisionType getDecisionType(ModRMDecision &decision) { satisfiesOneEntry = false; if (((index & 0xc0) == 0xc0) && - (decision.instructionIDs[index] != decision.instructionIDs[0xc0])) + (decision.instructionIDs[index] != decision.instructionIDs[0xc0])) satisfiesSplitRM = false; if (((index & 0xc0) != 0xc0) && - (decision.instructionIDs[index] != decision.instructionIDs[0x00])) + (decision.instructionIDs[index] != decision.instructionIDs[0x00])) satisfiesSplitRM = false; - if (((index & 0xc0) == 0xc0) && - (decision.instructionIDs[index] != decision.instructionIDs[index&0xf8])) + if (((index & 0xc0) == 0xc0) && (decision.instructionIDs[index] != + decision.instructionIDs[index & 0xf8])) satisfiesSplitReg = false; - if (((index & 0xc0) != 0xc0) && - (decision.instructionIDs[index] != decision.instructionIDs[index&0x38])) + if (((index & 0xc0) != 0xc0) && (decision.instructionIDs[index] != + decision.instructionIDs[index & 0x38])) satisfiesSplitMisc = false; } @@ -642,11 +646,13 @@ static ModRMDecisionType getDecisionType(ModRMDecision &decision) { /// @param dt - The decision type. /// @return - A pointer to the statically-allocated string (e.g., /// "MODRM_ONEENTRY" for MODRM_ONEENTRY). -static const char* stringForDecisionType(ModRMDecisionType dt) { -#define ENUM_ENTRY(n) case n: return #n; +static const char *stringForDecisionType(ModRMDecisionType dt) { +#define ENUM_ENTRY(n) \ + case n: \ + return #n; switch (dt) { - default: - llvm_unreachable("Unknown decision type"); + default: + llvm_unreachable("Unknown decision type"); MODRMTYPES }; #undef ENUM_ENTRY @@ -659,8 +665,7 @@ DisassemblerTables::DisassemblerTables() { HasConflicts = false; } -DisassemblerTables::~DisassemblerTables() { -} +DisassemblerTables::~DisassemblerTables() {} void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, unsigned &i1, unsigned &i2, @@ -678,31 +683,31 @@ void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, std::vector ModRMDecision; switch (dt) { - default: - llvm_unreachable("Unknown decision type"); - case MODRM_ONEENTRY: - ModRMDecision.push_back(decision.instructionIDs[0]); - break; - case MODRM_SPLITRM: - ModRMDecision.push_back(decision.instructionIDs[0x00]); - ModRMDecision.push_back(decision.instructionIDs[0xc0]); - break; - case MODRM_SPLITREG: - for (unsigned index = 0; index < 64; index += 8) - ModRMDecision.push_back(decision.instructionIDs[index]); - for (unsigned index = 0xc0; index < 256; index += 8) - ModRMDecision.push_back(decision.instructionIDs[index]); - break; - case MODRM_SPLITMISC: - for (unsigned index = 0; index < 64; index += 8) - ModRMDecision.push_back(decision.instructionIDs[index]); - for (unsigned index = 0xc0; index < 256; ++index) - ModRMDecision.push_back(decision.instructionIDs[index]); - break; - case MODRM_FULL: - for (unsigned short InstructionID : decision.instructionIDs) - ModRMDecision.push_back(InstructionID); - break; + default: + llvm_unreachable("Unknown decision type"); + case MODRM_ONEENTRY: + ModRMDecision.push_back(decision.instructionIDs[0]); + break; + case MODRM_SPLITRM: + ModRMDecision.push_back(decision.instructionIDs[0x00]); + ModRMDecision.push_back(decision.instructionIDs[0xc0]); + break; + case MODRM_SPLITREG: + for (unsigned index = 0; index < 64; index += 8) + ModRMDecision.push_back(decision.instructionIDs[index]); + for (unsigned index = 0xc0; index < 256; index += 8) + ModRMDecision.push_back(decision.instructionIDs[index]); + break; + case MODRM_SPLITMISC: + for (unsigned index = 0; index < 64; index += 8) + ModRMDecision.push_back(decision.instructionIDs[index]); + for (unsigned index = 0xc0; index < 256; ++index) + ModRMDecision.push_back(decision.instructionIDs[index]); + break; + case MODRM_FULL: + for (unsigned short InstructionID : decision.instructionIDs) + ModRMDecision.push_back(InstructionID); + break; } unsigned &EntryNumber = ModRMTable[ModRMDecision]; @@ -722,23 +727,23 @@ void DisassemblerTables::emitModRMDecision(raw_ostream &o1, raw_ostream &o2, o2 << "{" << stringForDecisionType(dt) << ", " << EntryNumber << "}"; switch (dt) { - default: - llvm_unreachable("Unknown decision type"); - case MODRM_ONEENTRY: - sEntryNumber += 1; - break; - case MODRM_SPLITRM: - sEntryNumber += 2; - break; - case MODRM_SPLITREG: - sEntryNumber += 16; - break; - case MODRM_SPLITMISC: - sEntryNumber += 8 + 64; - break; - case MODRM_FULL: - sEntryNumber += 256; - break; + default: + llvm_unreachable("Unknown decision type"); + case MODRM_ONEENTRY: + sEntryNumber += 1; + break; + case MODRM_SPLITRM: + sEntryNumber += 2; + break; + case MODRM_SPLITREG: + sEntryNumber += 16; + break; + case MODRM_SPLITMISC: + sEntryNumber += 8 + 64; + break; + case MODRM_FULL: + sEntryNumber += 256; + break; } // We assume that the index can fit into uint16_t. @@ -791,8 +796,9 @@ void DisassemblerTables::emitContextDecision(raw_ostream &o1, raw_ostream &o2, unsigned &i1, unsigned &i2, unsigned &ModRMTableNum, ContextDecision &decision, - const char* name) const { - o2.indent(i2) << "static const struct ContextDecision " << name << " = {{/* opcodeDecisions */\n"; + const char *name) const { + o2.indent(i2) << "static const struct ContextDecision " << name + << " = {{/* opcodeDecisions */\n"; i2++; for (unsigned index = 0; index < IC_max; ++index) { @@ -815,8 +821,8 @@ void DisassemblerTables::emitInstructionInfo(raw_ostream &o, o << "static const struct OperandSpecifier x86OperandSets[][" << X86_MAX_OPERANDS << "] = {\n"; - typedef SmallVector, - X86_MAX_OPERANDS> OperandListTy; + typedef SmallVector, X86_MAX_OPERANDS> + OperandListTy; std::map OperandSets; unsigned OperandSetNum = 0; @@ -829,14 +835,15 @@ void DisassemblerTables::emitInstructionInfo(raw_ostream &o, OperandList.push_back(std::make_pair(Encoding, Type)); } unsigned &N = OperandSets[OperandList]; - if (N != 0) continue; + if (N != 0) + continue; N = ++OperandSetNum; o << " { /* " << (OperandSetNum - 1) << " */\n"; for (unsigned i = 0, e = OperandList.size(); i != e; ++i) { const char *Encoding = stringForOperandEncoding(OperandList[i].first); - const char *Type = stringForOperandType(OperandList[i].second); + const char *Type = stringForOperandType(OperandList[i].second); o << " { " << Encoding << ", " << Type << " },\n"; } o << " },\n"; @@ -871,8 +878,8 @@ void DisassemblerTables::emitInstructionInfo(raw_ostream &o, } void DisassemblerTables::emitContextTable(raw_ostream &o, unsigned &i) const { - o.indent(i * 2) << "static const uint8_t " CONTEXTS_STR - "[" << ATTR_max << "] = {\n"; + o.indent(i * 2) << "static const uint8_t " CONTEXTS_STR "[" << ATTR_max + << "] = {\n"; i++; for (unsigned index = 0; index < ATTR_max; ++index) { @@ -908,8 +915,7 @@ void DisassemblerTables::emitContextTable(raw_ostream &o, unsigned &i) const { if (index & ATTR_EVEXB) o << "_B"; } - } - else if ((index & ATTR_64BIT) && (index & ATTR_REX2)) + } else if ((index & ATTR_64BIT) && (index & ATTR_REX2)) o << "IC_64BIT_REX2"; else if ((index & ATTR_64BIT) && (index & ATTR_REXW) && (index & ATTR_XS)) o << "IC_64BIT_REXW_XS"; @@ -977,12 +983,15 @@ void DisassemblerTables::emitContextDecisions(raw_ostream &o1, raw_ostream &o2, unsigned &ModRMTableNum) const { emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[0], ONEBYTE_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[1], TWOBYTE_STR); - emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[2], THREEBYTE38_STR); - emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[3], THREEBYTE3A_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[2], + THREEBYTE38_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[3], + THREEBYTE3A_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[4], XOP8_MAP_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[5], XOP9_MAP_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[6], XOPA_MAP_STR); - emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[7], THREEDNOW_MAP_STR); + emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[7], + THREEDNOW_MAP_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[8], MAP4_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[9], MAP5_STR); emitContextDecision(o1, o2, i1, i2, ModRMTableNum, *Tables[10], MAP6_STR); @@ -1026,24 +1035,22 @@ void DisassemblerTables::emit(raw_ostream &o) const { o << "\n"; } -void DisassemblerTables::setTableFields(ModRMDecision &decision, - const ModRMFilter &filter, - InstrUID uid, - uint8_t opcode) { +void DisassemblerTables::setTableFields(ModRMDecision &decision, + const ModRMFilter &filter, InstrUID uid, + uint8_t opcode) { for (unsigned index = 0; index < 256; ++index) { if (filter.accepts(index)) { if (decision.instructionIDs[index] == uid) continue; if (decision.instructionIDs[index] != 0) { - InstructionSpecifier &newInfo = - InstructionSpecifiers[uid]; + InstructionSpecifier &newInfo = InstructionSpecifiers[uid]; InstructionSpecifier &previousInfo = - InstructionSpecifiers[decision.instructionIDs[index]]; + InstructionSpecifiers[decision.instructionIDs[index]]; - if(previousInfo.name == "NOOP" && (newInfo.name == "XCHG16ar" || - newInfo.name == "XCHG32ar" || - newInfo.name == "XCHG64ar")) + if (previousInfo.name == "NOOP" && + (newInfo.name == "XCHG16ar" || newInfo.name == "XCHG32ar" || + newInfo.name == "XCHG64ar")) continue; // special case for XCHG*ar and NOOP if (outranks(previousInfo.insnContext, newInfo.insnContext)) @@ -1065,16 +1072,10 @@ void DisassemblerTables::setTableFields(ModRMDecision &decision, } } -void DisassemblerTables::setTableFields(OpcodeType type, - InstructionContext insnContext, - uint8_t opcode, - const ModRMFilter &filter, - InstrUID uid, - bool is32bit, - bool noPrefix, - bool ignoresVEX_L, - bool ignoresW, - unsigned addressSize) { +void DisassemblerTables::setTableFields( + OpcodeType type, InstructionContext insnContext, uint8_t opcode, + const ModRMFilter &filter, InstrUID uid, bool is32bit, bool noPrefix, + bool ignoresVEX_L, bool ignoresW, unsigned addressSize) { ContextDecision &decision = *Tables[type]; for (unsigned index = 0; index < IC_max; ++index) { @@ -1087,8 +1088,6 @@ void DisassemblerTables::setTableFields(OpcodeType type, InstructionSpecifiers[uid].insnContext, noPrefix, ignoresVEX_L, ignoresW, adSize64)) setTableFields(decision.opcodeDecisions[index].modRMDecisions[opcode], - filter, - uid, - opcode); + filter, uid, opcode); } } diff --git a/llvm/utils/git/code-format-helper.py b/llvm/utils/git/code-format-helper.py old mode 100644 new mode 100755 index 8876ba7a14de..697a27ab82a7 --- a/llvm/utils/git/code-format-helper.py +++ b/llvm/utils/git/code-format-helper.py @@ -1,21 +1,58 @@ #!/usr/bin/env python3 # -# ====- code-format-helper, runs code formatters from the ci --*- python -*--==# +# ====- code-format-helper, runs code formatters from the ci or in a hook --*- python -*--==# # # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. # See https://llvm.org/LICENSE.txt for license information. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception # -# ==-------------------------------------------------------------------------==# +# ==--------------------------------------------------------------------------------------==# import argparse import os import subprocess import sys -from functools import cached_property +from typing import List, Optional -import github -from github import IssueComment, PullRequest +""" +This script is run by GitHub actions to ensure that the code in PR's conform to +the coding style of LLVM. It can also be installed as a pre-commit git hook to +check the coding style before submitting it. The canonical source of this script +is in the LLVM source tree under llvm/utils/git. + +For C/C++ code it uses clang-format and for Python code it uses darker (which +in turn invokes black). + +You can learn more about the LLVM coding style on llvm.org: +https://llvm.org/docs/CodingStandards.html + +You can install this script as a git hook by symlinking it to the .git/hooks +directory: + +ln -s $(pwd)/llvm/utils/git/code-format-helper.py .git/hooks/pre-commit + +You can control the exact path to clang-format or darker with the following +environment variables: $CLANG_FORMAT_PATH and $DARKER_FORMAT_PATH. +""" + + +class FormatArgs: + start_rev: str = None + end_rev: str = None + repo: str = None + changed_files: List[str] = [] + token: str = None + verbose: bool = True + issue_number: int = 0 + + def __init__(self, args: argparse.Namespace = None) -> None: + if not args is None: + self.start_rev = args.start_rev + self.end_rev = args.end_rev + self.repo = args.repo + self.token = args.token + self.changed_files = args.changed_files + self.issue_number = args.issue_number class FormatHelper: @@ -31,9 +68,10 @@ class FormatHelper: def instructions(self) -> str: raise NotImplementedError() - def format_run( - self, changed_files: list[str], args: argparse.Namespace - ) -> str | None: + def has_tool(self) -> bool: + raise NotImplementedError() + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: raise NotImplementedError() def pr_comment_text_for_diff(self, diff: str) -> str: @@ -63,17 +101,18 @@ View the diff from {self.name} here. """ - def find_comment( - self, pr: PullRequest.PullRequest - ) -> IssueComment.IssueComment | None: + # TODO: any type should be replaced with the correct github type, but it requires refactoring to + # not require the github module to be installed everywhere. + def find_comment(self, pr: any) -> any: for comment in pr.as_issue().get_comments(): if self.comment_tag in comment.body: return comment return None - def update_pr( - self, comment_text: str, args: argparse.Namespace, create_new: bool - ) -> None: + def update_pr(self, comment_text: str, args: FormatArgs, create_new: bool) -> None: + import github + from github import IssueComment, PullRequest + repo = github.Github(args.token).get_repo(args.repo) pr = repo.get_issue(args.issue_number).as_pull_request() @@ -85,17 +124,25 @@ View the diff from {self.name} here. elif create_new: pr.as_issue().create_comment(comment_text) - def run(self, changed_files: list[str], args: argparse.Namespace) -> bool: + def run(self, changed_files: List[str], args: FormatArgs) -> bool: diff = self.format_run(changed_files, args) + should_update_gh = args.token is not None and args.repo is not None + if diff is None: - comment_text = f""" -:white_check_mark: With the latest revision this PR passed the {self.friendly_name}. -""" - self.update_pr(comment_text, args, create_new=False) + if should_update_gh: + comment_text = f""" + :white_check_mark: With the latest revision this PR passed the {self.friendly_name}. + """ + self.update_pr(comment_text, args, create_new=False) return True elif len(diff) > 0: - comment_text = self.pr_comment_text_for_diff(diff) - self.update_pr(comment_text, args, create_new=True) + if should_update_gh: + comment_text = self.pr_comment_text_for_diff(diff) + self.update_pr(comment_text, args, create_new=True) + else: + print( + f"Warning: {self.friendly_name}, {self.name} detected some issues with your code formatting..." + ) return False else: # The formatter failed but didn't output a diff (e.g. some sort of @@ -118,7 +165,7 @@ class ClangFormatHelper(FormatHelper): def should_include_extensionless_file(self, path: str) -> bool: return path.startswith("libcxx/include") - def filter_changed_files(self, changed_files: list[str]) -> list[str]: + def filter_changed_files(self, changed_files: List[str]) -> List[str]: filtered_files = [] for path in changed_files: _, ext = os.path.splitext(path) @@ -128,32 +175,49 @@ class ClangFormatHelper(FormatHelper): filtered_files.append(path) return filtered_files - def format_run( - self, changed_files: list[str], args: argparse.Namespace - ) -> str | None: + @property + def clang_fmt_path(self) -> str: + if "CLANG_FORMAT_PATH" in os.environ: + return os.environ["CLANG_FORMAT_PATH"] + return "git-clang-format" + + def has_tool(self) -> bool: + cmd = [self.clang_fmt_path, "-h"] + proc = None + try: + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except: + return False + return proc.returncode == 0 + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: cpp_files = self.filter_changed_files(changed_files) if not cpp_files: return None - cf_cmd = [ - "git-clang-format", - "--diff", - args.start_rev, - args.end_rev, - "--", - ] + cpp_files - print(f"Running: {' '.join(cf_cmd)}") + + cf_cmd = [self.clang_fmt_path, "--diff"] + + if args.start_rev and args.end_rev: + cf_cmd.append(args.start_rev) + cf_cmd.append(args.end_rev) + + cf_cmd.append("--") + cf_cmd += cpp_files + + if args.verbose: + print(f"Running: {' '.join(cf_cmd)}") self.cf_cmd = cf_cmd - proc = subprocess.run(cf_cmd, capture_output=True) + proc = subprocess.run(cf_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) sys.stdout.write(proc.stderr.decode("utf-8")) if proc.returncode != 0: # formatting needed, or the command otherwise failed - print(f"error: {self.name} exited with code {proc.returncode}") - # Print the diff in the log so that it is viewable there - print(proc.stdout.decode("utf-8")) + if args.verbose: + print(f"error: {self.name} exited with code {proc.returncode}") + # Print the diff in the log so that it is viewable there + print(proc.stdout.decode("utf-8")) return proc.stdout.decode("utf-8") else: - sys.stdout.write(proc.stdout.decode("utf-8")) return None @@ -165,7 +229,7 @@ class DarkerFormatHelper(FormatHelper): def instructions(self) -> str: return " ".join(self.darker_cmd) - def filter_changed_files(self, changed_files: list[str]) -> list[str]: + def filter_changed_files(self, changed_files: List[str]) -> List[str]: filtered_files = [] for path in changed_files: name, ext = os.path.splitext(path) @@ -174,29 +238,48 @@ class DarkerFormatHelper(FormatHelper): return filtered_files - def format_run( - self, changed_files: list[str], args: argparse.Namespace - ) -> str | None: + @property + def darker_fmt_path(self) -> str: + if "DARKER_FORMAT_PATH" in os.environ: + return os.environ["DARKER_FORMAT_PATH"] + return "darker" + + def has_tool(self) -> bool: + cmd = [self.darker_fmt_path, "--version"] + proc = None + try: + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + except: + return False + return proc.returncode == 0 + + def format_run(self, changed_files: List[str], args: FormatArgs) -> Optional[str]: py_files = self.filter_changed_files(changed_files) if not py_files: return None darker_cmd = [ - "darker", + self.darker_fmt_path, "--check", "--diff", - "-r", - f"{args.start_rev}..{args.end_rev}", - ] + py_files - print(f"Running: {' '.join(darker_cmd)}") + ] + if args.start_rev and args.end_rev: + darker_cmd += ["-r", f"{args.start_rev}...{args.end_rev}"] + darker_cmd += py_files + if args.verbose: + print(f"Running: {' '.join(darker_cmd)}") self.darker_cmd = darker_cmd - proc = subprocess.run(darker_cmd, capture_output=True) - sys.stdout.write(proc.stderr.decode("utf-8")) + proc = subprocess.run( + darker_cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + if args.verbose: + sys.stdout.write(proc.stderr.decode("utf-8")) if proc.returncode != 0: # formatting needed, or the command otherwise failed - print(f"error: {self.name} exited with code {proc.returncode}") - # Print the diff in the log so that it is viewable there - print(proc.stdout.decode("utf-8")) + if args.verbose: + print(f"error: {self.name} exited with code {proc.returncode}") + # Print the diff in the log so that it is viewable there + print(proc.stdout.decode("utf-8")) return proc.stdout.decode("utf-8") else: sys.stdout.write(proc.stdout.decode("utf-8")) @@ -205,7 +288,39 @@ class DarkerFormatHelper(FormatHelper): ALL_FORMATTERS = (DarkerFormatHelper(), ClangFormatHelper()) + +def hook_main(): + # fill out args + args = FormatArgs() + args.verbose = False + + # find the changed files + cmd = ["git", "diff", "--cached", "--name-only", "--diff-filter=d"] + proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output = proc.stdout.decode("utf-8") + for line in output.splitlines(): + args.changed_files.append(line) + + failed_fmts = [] + for fmt in ALL_FORMATTERS: + if fmt.has_tool(): + if not fmt.run(args.changed_files, args): + failed_fmts.append(fmt.name) + else: + print(f"Couldn't find {fmt.name}, can't check " + fmt.friendly_name.lower()) + + if len(failed_fmts) > 0: + sys.exit(1) + + sys.exit(0) + + if __name__ == "__main__": + script_path = os.path.abspath(__file__) + if ".git/hooks" in script_path: + hook_main() + sys.exit(0) + parser = argparse.ArgumentParser() parser.add_argument( "--token", type=str, required=True, help="GitHub authentiation token" @@ -232,7 +347,7 @@ if __name__ == "__main__": help="Comma separated list of files that has been changed", ) - args = parser.parse_args() + args = FormatArgs(parser.parse_args()) changed_files = [] if args.changed_files: diff --git a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/hicpp/BUILD.gn b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/hicpp/BUILD.gn index cd886d89afcf..cb0e823b3acb 100644 --- a/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/hicpp/BUILD.gn +++ b/llvm/utils/gn/secondary/clang-tools-extra/clang-tidy/hicpp/BUILD.gn @@ -21,6 +21,7 @@ static_library("hicpp") { sources = [ "ExceptionBaseclassCheck.cpp", "HICPPTidyModule.cpp", + "IgnoredRemoveResultCheck.cpp", "MultiwayPathsCoveredCheck.cpp", "NoAssemblerCheck.cpp", "SignedBitwiseCheck.cpp", diff --git a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn index 64d38360e250..1d3fc6cfdfaa 100644 --- a/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/ExecutionEngine/Orc/Debugging/BUILD.gn @@ -1,9 +1,9 @@ static_library("Debugging") { output_name = "LLVMOrcDebugging" deps = [ + "//llvm/lib/DebugInfo/DWARF", "//llvm/lib/ExecutionEngine/Orc", "//llvm/lib/ExecutionEngine/Orc/Shared", - "//llvm/lib/DebugInfo/DWARF", "//llvm/lib/Support", "//llvm/lib/TargetParser", ] @@ -11,6 +11,7 @@ static_library("Debugging") { "DebugInfoSupport.cpp", "DebuggerSupport.cpp", "DebuggerSupportPlugin.cpp", + "LLJITUtilsCBindings.cpp", "PerfSupportPlugin.cpp", ] if (current_os == "linux") { diff --git a/llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn b/llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn index 2e5b7e03bd65..668512ecba88 100644 --- a/llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/lib/Target/BPF/BUILD.gn @@ -80,6 +80,10 @@ static_library("LLVMBPFCodeGen") { "BPFSubtarget.cpp", "BPFTargetMachine.cpp", "BTFDebug.cpp", + "GISel/BPFCallLowering.cpp", + "GISel/BPFInstructionSelector.cpp", + "GISel/BPFLegalizerInfo.cpp", + "GISel/BPFRegisterBankInfo.cpp", ] } diff --git a/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn b/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn index c258b5232280..47c7bb3d97d1 100644 --- a/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn +++ b/llvm/utils/gn/secondary/llvm/unittests/ExecutionEngine/Orc/BUILD.gn @@ -4,6 +4,7 @@ unittest("OrcJITTests") { deps = [ "//llvm/lib/ExecutionEngine", "//llvm/lib/ExecutionEngine/Orc", + "//llvm/lib/ExecutionEngine/Orc/Debugging", "//llvm/lib/ExecutionEngine/Orc/Shared", "//llvm/lib/ExecutionEngine/RuntimeDyld", "//llvm/lib/IR", diff --git a/mlir/cmake/modules/MLIRConfig.cmake.in b/mlir/cmake/modules/MLIRConfig.cmake.in index cd8cec5dedf3..d4da3cd98cce 100644 --- a/mlir/cmake/modules/MLIRConfig.cmake.in +++ b/mlir/cmake/modules/MLIRConfig.cmake.in @@ -13,6 +13,7 @@ set(MLIR_TABLEGEN_EXE "@MLIR_CONFIG_TABLEGEN_EXE@") set(MLIR_PDLL_TABLEGEN_EXE "@MLIR_CONFIG_PDLL_TABLEGEN_EXE@") set(MLIR_INSTALL_AGGREGATE_OBJECTS "@MLIR_INSTALL_AGGREGATE_OBJECTS@") set(MLIR_ENABLE_BINDINGS_PYTHON "@MLIR_ENABLE_BINDINGS_PYTHON@") +set(MLIR_ENABLE_EXECUTION_ENGINE "@MLIR_ENABLE_EXECUTION_ENGINE@") # For mlir_tablegen() set(MLIR_INCLUDE_DIR "@MLIR_INCLUDE_DIR@") diff --git a/mlir/include/mlir-c/IR.h b/mlir/include/mlir-c/IR.h index 413eaa6aa3fe..82da511f807a 100644 --- a/mlir/include/mlir-c/IR.h +++ b/mlir/include/mlir-c/IR.h @@ -940,6 +940,9 @@ MLIR_CAPI_EXPORTED void mlirValueReplaceAllUsesOfWith(MlirValue of, /// Returns whether the op operand is null. MLIR_CAPI_EXPORTED bool mlirOpOperandIsNull(MlirOpOperand opOperand); +/// Returns the value of an op operand. +MLIR_CAPI_EXPORTED MlirValue mlirOpOperandGetValue(MlirOpOperand opOperand); + /// Returns the owner operation of an op operand. MLIR_CAPI_EXPORTED MlirOperation mlirOpOperandGetOwner(MlirOpOperand opOperand); diff --git a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td index f9578cf37d5d..c638646b9c32 100644 --- a/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td +++ b/mlir/include/mlir/Dialect/Affine/IR/AffineOps.td @@ -16,6 +16,7 @@ include "mlir/Dialect/Arith/IR/ArithBase.td" include "mlir/Dialect/Affine/IR/AffineMemoryOpInterfaces.td" include "mlir/Interfaces/ControlFlowInterfaces.td" +include "mlir/Interfaces/InferTypeOpInterface.td" include "mlir/Interfaces/LoopLikeInterface.td" include "mlir/Interfaces/SideEffectInterfaces.td" @@ -63,10 +64,6 @@ def AffineApplyOp : Affine_Op<"apply", [Pure]> { // has a constant builder. That way we wouldn't need to explicitly specify the // result types here. let builders = [ - OpBuilder<(ins "AffineMap":$map, "ValueRange":$mapOperands), - [{ - build($_builder, $_state, $_builder.getIndexType(), map, mapOperands); - }]>, OpBuilder<(ins "ArrayRef ":$exprList,"ValueRange":$mapOperands), [{ build($_builder, $_state, $_builder.getIndexType(), @@ -541,13 +538,6 @@ class AffineMinMaxOpBase traits = []> : let arguments = (ins AffineMapAttr:$map, Variadic:$operands); let results = (outs Index); - let builders = [ - OpBuilder<(ins "AffineMap":$affineMap, "ValueRange":$mapOperands), - [{ - build($_builder, $_state, $_builder.getIndexType(), affineMap, mapOperands); - }]> - ]; - let extraClassDeclaration = [{ static StringRef getMapAttrStrName() { return "map"; } AffineMap getAffineMap() { return getMap(); } @@ -1068,7 +1058,7 @@ def AffineVectorStoreOp : AffineStoreOpBase<"vector_store"> { //===----------------------------------------------------------------------===// def AffineDelinearizeIndexOp : Affine_Op<"delinearize_index", - [Pure]> { + [Pure, DeclareOpInterfaceMethods]> { let summary = "delinearize an index"; let description = [{ The `affine.delinearize_index` operation takes a single index value and diff --git a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td index 02e0b7980c10..6d133d69dd0f 100644 --- a/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td +++ b/mlir/include/mlir/Dialect/Arith/IR/ArithOps.td @@ -195,6 +195,12 @@ def Arith_ConstantOp : Op { let summary = "integer addition operation"; let description = [{ + Performs N-bit addition on the operands. The operands are interpreted as + unsigned bitvectors. The result is represented by a bitvector containing the + mathematical value of the addition modulo 2^n, where `n` is the bitwidth. + Because `arith` integers use a two's complement representation, this operation + is applicable on both signed and unsigned integer operands. + The `addi` operation takes two operands and returns one result, each of these is required to be the same type. This type may be an integer scalar type, a vector whose element type is integer, or a tensor of integers. It has no @@ -230,7 +236,7 @@ def Arith_AddUIExtendedOp : Arith_Op<"addui_extended", [Pure, Commutative, let description = [{ Performs (N+1)-bit addition on zero-extended operands. Returns two results: the N-bit sum (same type as both operands), and the overflow bit - (boolean-like), where`1` indicates unsigned addition overflow, while `0` + (boolean-like), where `1` indicates unsigned addition overflow, while `0` indicates no overflow. Example: @@ -276,6 +282,18 @@ def Arith_SubIOp : Arith_TotalIntBinaryOp<"subi"> { let summary = [{ Integer subtraction operation. }]; + let description = [{ + Performs N-bit subtraction on the operands. The operands are interpreted as unsigned + bitvectors. The result is represented by a bitvector containing the mathematical + value of the subtraction modulo 2^n, where `n` is the bitwidth. Because `arith` + integers use a two's complement representation, this operation is applicable on + both signed and unsigned integer operands. + + The `subi` operation takes two operands and returns one result, each of + these is required to be the same type. This type may be an integer scalar type, + a vector whose element type is integer, or a tensor of integers. It has no + standard attributes. + }]; let hasFolder = 1; let hasCanonicalizer = 1; } @@ -288,6 +306,18 @@ def Arith_MulIOp : Arith_TotalIntBinaryOp<"muli", [Commutative]> { let summary = [{ Integer multiplication operation. }]; + let description = [{ + Performs N-bit multiplication on the operands. The operands are interpreted as + unsigned bitvectors. The result is represented by a bitvector containing the + mathematical value of the multiplication modulo 2^n, where `n` is the bitwidth. + Because `arith` integers use a two's complement representation, this operation is + applicable on both signed and unsigned integer operands. + + The `muli` operation takes two operands and returns one result, each of + these is required to be the same type. This type may be an integer scalar type, + a vector whose element type is integer, or a tensor of integers. It has no + standard attributes. + }]; let hasFolder = 1; let hasCanonicalizer = 1; } diff --git a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td index 7b9c74e0b8f6..02238f0a18ba 100644 --- a/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td +++ b/mlir/include/mlir/Dialect/ArmSME/Transforms/Passes.td @@ -87,8 +87,9 @@ def TileAllocation let summary = "Allocate SME tiles"; let description = [{ This pass does tile allocation for SME "virtual tiles". It is run at the - 'func.func' op level, replacing 'arm_sme.get_tile_id' ops with (i32) tile - ids. An error will be emitted when there's no tiles left. + 'func.func' op level, and assigns tile IDs (via an attribute) to all ops + that implement the `ArmSMETileOpInterface`. An error will be emitted when + there's no tiles left. }]; let constructor = "mlir::arm_sme::createTileAllocationPass()"; let dependentDialects = ["func::FuncDialect"]; diff --git a/mlir/include/mlir/Dialect/Complex/IR/ComplexOps.td b/mlir/include/mlir/Dialect/Complex/IR/ComplexOps.td index ada6c14b5b71..e19d714cadf8 100644 --- a/mlir/include/mlir/Dialect/Complex/IR/ComplexOps.td +++ b/mlir/include/mlir/Dialect/Complex/IR/ComplexOps.td @@ -145,7 +145,7 @@ def ConstantOp : Complex_Op<"constant", [ }]; let arguments = (ins ArrayAttr:$value); - let results = (outs Complex:$complex); + let results = (outs AnyComplex:$complex); let assemblyFormat = "$value attr-dict `:` type($complex)"; let hasFolder = 1; diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td index 3b5984498cf8..a7b269eb41ee 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMEnums.td @@ -428,6 +428,29 @@ def DISubprogramFlags : I32BitEnumAttr< let printBitEnumPrimaryGroups = 1; } +//===----------------------------------------------------------------------===// +// IntegerOverflowFlags +//===----------------------------------------------------------------------===// + +def IOFnone : I32BitEnumAttrCaseNone<"none">; +def IOFnsw : I32BitEnumAttrCaseBit<"nsw", 0>; +def IOFnuw : I32BitEnumAttrCaseBit<"nuw", 1>; + +def IntegerOverflowFlags : I32BitEnumAttr< + "IntegerOverflowFlags", + "LLVM integer overflow flags", + [IOFnone, IOFnsw, IOFnuw]> { + let separator = ", "; + let cppNamespace = "::mlir::LLVM"; + let genSpecializedAttr = 0; + let printBitEnumPrimaryGroups = 1; +} + +def LLVM_IntegerOverflowFlagsAttr : + EnumAttr { + let assemblyFormat = "`<` $value `>`"; +} + //===----------------------------------------------------------------------===// // FastmathFlags //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td index c5d65f792254..81589eaf5fd0 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMInterfaces.td @@ -48,6 +48,63 @@ def FastmathFlagsInterface : OpInterface<"FastmathFlagsInterface"> { ]; } +def IntegerOverflowFlagsInterface : OpInterface<"IntegerOverflowFlagsInterface"> { + let description = [{ + Access to op integer overflow flags. + }]; + + let cppNamespace = "::mlir::LLVM"; + + let methods = [ + InterfaceMethod< + /*desc=*/ "Returns an IntegerOverflowFlagsAttr attribute for the operation", + /*returnType=*/ "IntegerOverflowFlagsAttr", + /*methodName=*/ "getOverflowAttr", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + auto op = cast(this->getOperation()); + return op.getOverflowFlagsAttr(); + }] + >, + InterfaceMethod< + /*desc=*/ "Returns whether the operation has the No Unsigned Wrap keyword", + /*returnType=*/ "bool", + /*methodName=*/ "hasNoUnsignedWrap", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + auto op = cast(this->getOperation()); + IntegerOverflowFlags flags = op.getOverflowFlagsAttr().getValue(); + return bitEnumContainsAll(flags, IntegerOverflowFlags::nuw); + }] + >, + InterfaceMethod< + /*desc=*/ "Returns whether the operation has the No Signed Wrap keyword", + /*returnType=*/ "bool", + /*methodName=*/ "hasNoSignedWrap", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + auto op = cast(this->getOperation()); + IntegerOverflowFlags flags = op.getOverflowFlagsAttr().getValue(); + return bitEnumContainsAll(flags, IntegerOverflowFlags::nsw); + }] + >, + StaticInterfaceMethod< + /*desc=*/ [{Returns the name of the IntegerOveflowFlagsAttr attribute + for the operation}], + /*returnType=*/ "StringRef", + /*methodName=*/ "getIntegerOverflowAttrName", + /*args=*/ (ins), + /*methodBody=*/ [{}], + /*defaultImpl=*/ [{ + return "overflowFlags"; + }] + > + ]; +} + def BranchWeightOpInterface : OpInterface<"BranchWeightOpInterface"> { let description = [{ An interface for operations that can carry branch weights metadata. It diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td index 92460fa06f53..d7690b84807f 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMOps.td @@ -55,6 +55,26 @@ class LLVM_IntArithmeticOp($_location, $lhs, $rhs); }]; } +class LLVM_IntArithmeticOpWithOverflowFlag traits = []> : + LLVM_ArithmeticOpBase], traits)> { + dag iofArg = ( + ins DefaultValuedAttr:$overflowFlags); + let arguments = !con(commonArgs, iofArg); + string mlirBuilder = [{ + auto op = $_builder.create<$_qualCppClassName>($_location, $lhs, $rhs); + moduleImport.setIntegerOverflowFlagsAttr(inst, op); + $res = op; + }]; + let assemblyFormat = [{ + $lhs `,` $rhs (`overflow` `` $overflowFlags^)? + custom(attr-dict) `:` type($res) + }]; + string llvmBuilder = + "$res = builder.Create" # instName # + "($lhs, $rhs, /*Name=*/\"\", op.hasNoUnsignedWrap(), op.hasNoSignedWrap());"; +} class LLVM_FloatArithmeticOp traits = []> : LLVM_ArithmeticOpBase; -def LLVM_SubOp : LLVM_IntArithmeticOp<"sub", "Sub">; -def LLVM_MulOp : LLVM_IntArithmeticOp<"mul", "Mul", [Commutative]>; +def LLVM_AddOp : LLVM_IntArithmeticOpWithOverflowFlag<"add", "Add", + [Commutative]>; +def LLVM_SubOp : LLVM_IntArithmeticOpWithOverflowFlag<"sub", "Sub", []>; +def LLVM_MulOp : LLVM_IntArithmeticOpWithOverflowFlag<"mul", "Mul", + [Commutative]>; def LLVM_UDivOp : LLVM_IntArithmeticOp<"udiv", "UDiv">; def LLVM_SDivOp : LLVM_IntArithmeticOp<"sdiv", "SDiv">; def LLVM_URemOp : LLVM_IntArithmeticOp<"urem", "URem">; @@ -102,7 +124,7 @@ def LLVM_OrOp : LLVM_IntArithmeticOp<"or", "Or"> { let hasFolder = 1; } def LLVM_XOrOp : LLVM_IntArithmeticOp<"xor", "Xor">; -def LLVM_ShlOp : LLVM_IntArithmeticOp<"shl", "Shl"> { +def LLVM_ShlOp : LLVM_IntArithmeticOpWithOverflowFlag<"shl", "Shl", []> { let hasFolder = 1; } def LLVM_LShrOp : LLVM_IntArithmeticOp<"lshr", "LShr">; @@ -262,10 +284,10 @@ def LLVM_GEPOp : LLVM_Op<"getelementptr", [Pure, }]; let builders = [ - OpBuilder<(ins "Type":$resultType, "Type":$basePtrType, "Value":$basePtr, + OpBuilder<(ins "Type":$resultType, "Type":$elementType, "Value":$basePtr, "ValueRange":$indices, CArg<"bool", "false">:$inbounds, CArg<"ArrayRef", "{}">:$attributes)>, - OpBuilder<(ins "Type":$resultType, "Type":$basePtrType, "Value":$basePtr, + OpBuilder<(ins "Type":$resultType, "Type":$elementType, "Value":$basePtr, "ArrayRef":$indices, CArg<"bool", "false">:$inbounds, CArg<"ArrayRef", "{}">:$attributes)>, ]; diff --git a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td index de65f3176c46..77ed9db5e71b 100644 --- a/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td +++ b/mlir/include/mlir/Dialect/Linalg/TransformOps/LinalgTransformOps.td @@ -574,6 +574,11 @@ def MatchOp : Op:$ops, OptionalAttr:$interface, OptionalAttr:$op_attrs, - OptionalAttr:$filter_result_type); + OptionalAttr:$filter_result_type, + OptionalAttr:$filter_operand_types); // TODO: variadic results when needed. let results = (outs TransformHandleTypeInterface:$results); @@ -609,6 +615,7 @@ def MatchOp : Op replacements; + static llvm::SmallDenseSet + getPreservedProducerResults(GenericOp producer, GenericOp consumer); }; FailureOr fuseElementwiseOps(RewriterBase &rewriter, OpOperand *fusedOperand); diff --git a/mlir/include/mlir/Dialect/Math/IR/MathOps.td b/mlir/include/mlir/Dialect/Math/IR/MathOps.td index f8e9fd601304..9742d3d936df 100644 --- a/mlir/include/mlir/Dialect/Math/IR/MathOps.td +++ b/mlir/include/mlir/Dialect/Math/IR/MathOps.td @@ -300,6 +300,35 @@ def Math_CosOp : Math_FloatUnaryOp<"cos"> { let hasFolder = 1; } +//===----------------------------------------------------------------------===// +// AcosOp +//===----------------------------------------------------------------------===// + +def Math_AcosOp : Math_FloatUnaryOp<"acos"> { + let summary = "arcus cosine of the specified value"; + let description = [{ + Syntax: + + ``` + operation ::= ssa-id `=` `math.acos` ssa-use `:` type + ``` + + The `acos` operation computes the arcus cosine of a given value. It takes one + operand of floating point type (i.e., scalar, tensor or vector) and returns one + result of the same type. It has no standard attributes. + + Example: + + ```mlir + // Scalar arcus cosine value. + %a = math.acos %b : f64 + ``` + }]; + let hasFolder = 1; +} + + + //===----------------------------------------------------------------------===// // SinOp //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td index 361e67fd1e19..e6cdba949b17 100644 --- a/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td +++ b/mlir/include/mlir/Dialect/Mesh/IR/MeshOps.td @@ -40,26 +40,27 @@ def Mesh_ClusterOp : Mesh_Op<"cluster", [Symbol]> { determine the layout and the addressing space of the computation distributed across the mesh. - 3. `dim_sizes`: This attribute represents the device assignment along the - axes of the cluster. Each integer in the array corresponds to the number of - devices along a specific axis. If an integer value is 0, it implies that the - number of devices along that axis is unknown. This flexibility allows for - dynamic device assignment or configurations where the exact number of - devices might not be determined during compile time. + 3. `dim_sizes`: This attribute represents the shape of the device cluster. + It uses the same notation as a tensor shape. Also allowing for dynamic + dimensions. + This flexibility allows for dynamic device assignment or configurations + where the exact number of devices might not be determined during compile + time. + For example `2x?x4`. Example: ``` // A device mesh cluster with 3 axes, the total device number is 4 * 8 * 12 // The dimension sizes are 4, 8, 12 - mesh.cluster @mesh0(rank = 3, dim_sizes = [4, 8, 12]) + mesh.cluster @mesh0(rank = 3, dim_sizes = 4x8x12) // A device mesh cluster with 2 axes, the total device number is unknown // The first dimension size is 4 and the second is unknown - mesh.cluster @mesh1(rank = 2, dim_sizes = [4]) + mesh.cluster @mesh1(rank = 2, dim_sizes = 4) // A device mesh cluster with 2 axes, the total device number is unknown // The first dimension size is unknown and the second is 4 - mesh.cluster @mesh2(rank = 2, dim_sizes = [0, 4]) + mesh.cluster @mesh2(rank = 2, dim_sizes = ?x4) // A device mesh cluster with 2 axes, the number of devices along both axes // is unknown @@ -76,7 +77,7 @@ def Mesh_ClusterOp : Mesh_Op<"cluster", [Symbol]> { DefaultValuedAttr:$dim_sizes ); let assemblyFormat = [{ - $sym_name `(` `rank` `=` $rank (`,` `dim_sizes` `=` $dim_sizes^)? `)` + $sym_name `(` `rank` `=` $rank (`,` `dim_sizes` `=` custom($dim_sizes)^)? `)` attr-dict }]; let extraClassDeclaration = [{ @@ -88,7 +89,7 @@ def Mesh_ClusterOp : Mesh_Op<"cluster", [Symbol]> { template void canonicalDimSizes(OutIt outIt) { std::copy(getDimSizes().begin(), getDimSizes().end(), outIt); - std::fill_n(outIt, getRank() - getDimSizes().size(), 0); + std::fill_n(outIt, getRank() - getDimSizes().size(), ::mlir::ShapedType::kDynamic); } }]; let hasVerifier = 1; @@ -210,7 +211,7 @@ def Mesh_AllGatherOp : Mesh_CollectiveCommunicationOpBase<"all_gather", [ Example: ```mlir - mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 2]) + mesh.cluster @mesh0(rank = 2, dim_sizes = 2x2) ... %1 = mesh.all_gather %0 on @mesh0 mesh_axes = [1] gather_axis = 1 : tensor<2x2xi8> -> tensor<2x4xi8> @@ -295,7 +296,7 @@ def Mesh_AllToAllOp : Mesh_CollectiveCommunicationOpBase<"all_to_all", [ Example: ``` - mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) + mesh.cluster @mesh0(rank = 1, dim_sizes = 3) ... %1 = mesh.all_to_all %0 on @mesh0 mesh_axes = [0] split_axis = 0 concat_axis = 0 @@ -527,7 +528,7 @@ def Mesh_ReduceScatterOp : Mesh_CollectiveCommunicationOpBase<"reduce_scatter", across the device group. Example: ``` - mesh.cluster @mesh0(rank = 1, dim_sizes = [2, 2]) + mesh.cluster @mesh0(rank = 1, dim_sizes = 2x2) ... %1 = mesh.reduce_scatter %0 on @mesh0 mesh_axes = [1] reduction = scatter_axis = 0 diff --git a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td index 391e77e0c408..9d48b1f1c3f9 100644 --- a/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td +++ b/mlir/include/mlir/Dialect/OpenACC/OpenACCOps.td @@ -156,7 +156,7 @@ def DeclareActionAttr : OpenACC_Attr<"DeclareAction", "declare_action"> { } // Device type enumeration. -def OpenACC_DeviceTypeStar : I32EnumAttrCase<"Star", 0, "*">; +def OpenACC_DeviceTypeStar : I32EnumAttrCase<"Star", 0, "star">; def OpenACC_DeviceTypeDefault : I32EnumAttrCase<"Default", 1, "default">; def OpenACC_DeviceTypeHost : I32EnumAttrCase<"Host", 2, "host">; def OpenACC_DeviceTypeMulticore : I32EnumAttrCase<"Multicore", 3, "multicore">; diff --git a/mlir/include/mlir/Dialect/SCF/Transforms/Transforms.h b/mlir/include/mlir/Dialect/SCF/Transforms/Transforms.h index 347beb9e4c64..cad517359945 100644 --- a/mlir/include/mlir/Dialect/SCF/Transforms/Transforms.h +++ b/mlir/include/mlir/Dialect/SCF/Transforms/Transforms.h @@ -128,6 +128,13 @@ struct PipeliningOption { /// lambda to generate the predicated version of operations. bool peelEpilogue = true; + /// Control whether the transformation checks that the number of iterations is + /// greater or equal to the number of stages and skip the transformation if + /// this is not the case. If the loop is dynamic and this is set to true and + /// the loop bounds are not static the pipeliner will have to predicate + /// operations in the the prologue/epilogue. + bool supportDynamicLoops = false; + // Callback to predicate operations when the prologue or epilogue are not // peeled. This takes the original operation, an i1 predicate value and the // pattern rewriter. It is expected to replace the given operation with diff --git a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBitOps.td b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBitOps.td index 91b69576cd1d..b460c8e68aa0 100644 --- a/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBitOps.td +++ b/mlir/include/mlir/Dialect/SPIRV/IR/SPIRVBitOps.td @@ -334,6 +334,8 @@ def SPIRV_BitwiseXorOp : SPIRV_BitBinaryOp<"BitwiseXor", %2 = spirv.BitwiseXor %0, %1 : vector<4xi32> ``` }]; + + let hasFolder = 1; } // ----- @@ -368,6 +370,8 @@ def SPIRV_ShiftLeftLogicalOp : SPIRV_ShiftOp<"ShiftLeftLogical", %5 = spirv.ShiftLeftLogical %3, %4 : vector<3xi32>, vector<3xi16> ``` }]; + + let hasFolder = 1; } // ----- @@ -399,6 +403,8 @@ def SPIRV_ShiftRightArithmeticOp : SPIRV_ShiftOp<"ShiftRightArithmetic", %5 = spirv.ShiftRightArithmetic %3, %4 : vector<3xi32>, vector<3xi16> ``` }]; + + let hasFolder = 1; } // ----- @@ -431,6 +437,8 @@ def SPIRV_ShiftRightLogicalOp : SPIRV_ShiftOp<"ShiftRightLogical", %5 = spirv.ShiftRightLogical %3, %4 : vector<3xi32>, vector<3xi16> ``` }]; + + let hasFolder = 1; } // ----- diff --git a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h index 41ffc9299460..08d3bb157a0e 100644 --- a/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h +++ b/mlir/include/mlir/Dialect/Vector/Transforms/VectorTransforms.h @@ -59,16 +59,6 @@ struct VectorTransformsOptions { vectorTransferSplit = opt; return *this; } - - /// Option to control if vector.transpose can lower to a vector.shape_cast. - /// TODO: ATM it's not possible to lower `vector.shape_cast` to SPIR-V - /// and hence the need for this opt-out. Once the missing support has been - /// added, this option can be removed. - bool useShapeCast = true; - VectorTransformsOptions &setUseShapeCast(bool opt = true) { - useShapeCast = opt; - return *this; - } }; //===----------------------------------------------------------------------===// diff --git a/mlir/include/mlir/IR/BuiltinAttributes.td b/mlir/include/mlir/IR/BuiltinAttributes.td index be9cbec768d7..d9295936ee97 100644 --- a/mlir/include/mlir/IR/BuiltinAttributes.td +++ b/mlir/include/mlir/IR/BuiltinAttributes.td @@ -454,9 +454,15 @@ def Builtin_DenseResourceElementsAttr : Builtin_Attr<"DenseResourceElements", Examples: ```mlir - // A tensor referencing a builtin dialect resource, `resource_1`, with two - // unsigned i32 elements. - dense_resource : tensor<2xui32> + "example.user_op"() {attr = dense_resource : tensor<3xi64> } : () -> () + + {-# + dialect_resources: { + builtin: { + blob1: "0x08000000010000000000000002000000000000000300000000000000" + } + } + #-} ``` }]; let parameters = (ins diff --git a/mlir/include/mlir/IR/BuiltinLocationAttributes.td b/mlir/include/mlir/IR/BuiltinLocationAttributes.td index e1656f268795..dfcc180071f7 100644 --- a/mlir/include/mlir/IR/BuiltinLocationAttributes.td +++ b/mlir/include/mlir/IR/BuiltinLocationAttributes.td @@ -112,8 +112,8 @@ def FusedLoc : Builtin_LocationAttr<"FusedLoc"> { Syntax: ``` - fused-location ::= `fused` fusion-metadata? `[` location (location `,`)* `]` fusion-metadata ::= `<` attribute-value `>` + fused-location ::= `fused` fusion-metadata? `[` (location (`,` location)* )? `]` ``` An instance of a `fused` location represents a grouping of several other @@ -126,7 +126,7 @@ def FusedLoc : Builtin_LocationAttr<"FusedLoc"> { Example: ```mlir - loc(fused["mysource.cc":10:8, "mysource.cc":22:8) + loc(fused["mysource.cc":10:8, "mysource.cc":22:8]) loc(fused<"CSE">["mysource.cc":10:8, "mysource.cc":22:8]) ``` }]; @@ -159,7 +159,7 @@ def NameLoc : Builtin_LocationAttr<"NameLoc"> { This can be useful for representing the locations of variable, or node, definitions. - Example: + #### Example: ```mlir loc("CSE"("mysource.cc":10:8)) @@ -191,6 +191,13 @@ def OpaqueLoc : Builtin_LocationAttr<"OpaqueLoc"> { structure that is external to MLIR and an optional location that can be used if the first one is not suitable. Since it contains an external structure, only the optional location is used during serialization. + + #### Example: + + ```mlir + %0 = "example.operation"() : () -> i32 loc("mysource") + %1 = arith.constant 4 : index loc(callsite("mysum" at "mysource.cc":10:8)) + ``` }]; let parameters = (ins "uintptr_t":$underlyingLocation, "TypeID":$underlyingTypeID, diff --git a/mlir/include/mlir/IR/BuiltinTypes.td b/mlir/include/mlir/IR/BuiltinTypes.td index 1d7772810ae6..4cade83dd3c3 100644 --- a/mlir/include/mlir/IR/BuiltinTypes.td +++ b/mlir/include/mlir/IR/BuiltinTypes.td @@ -47,7 +47,7 @@ def Builtin_Complex : Builtin_Type<"Complex", "complex"> { element type, which is composed of a real and imaginary value of that element type. The element must be a floating point or integer scalar type. - Examples: + #### Example: ```mlir complex @@ -251,6 +251,16 @@ def Builtin_Function : Builtin_Type<"Function", "function"> { The function type can be thought of as a function signature. It consists of a list of formal parameter types and a list of formal result types. + + #### Example: + + ```mlir + func.func @add_one(%arg0 : i64) -> i64 { + %c1 = arith.constant 1 : i64 + %0 = arith.addi %arg0, %c1 : i64 + return %0 : i64 + } + ``` }]; let parameters = (ins "ArrayRef":$inputs, "ArrayRef":$results); let builders = [ @@ -392,10 +402,10 @@ def Builtin_MemRef : Builtin_Type<"MemRef", "memref", [ Syntax: ``` - memref-type ::= `memref` `<` dimension-list-ranked type - (`,` layout-specification)? (`,` memory-space)? `>` layout-specification ::= attribute-value memory-space ::= attribute-value + memref-type ::= `memref` `<` dimension-list-ranked type + (`,` layout-specification)? (`,` memory-space)? `>` ``` A `memref` type is a reference to a region of memory (similar to a buffer @@ -667,8 +677,23 @@ def Builtin_MemRef : Builtin_Type<"MemRef", "memref", [ def Builtin_None : Builtin_Type<"None", "none"> { let summary = "A unit type"; let description = [{ + Syntax: + + ``` + none-type ::= `none` + ``` + NoneType is a unit type, i.e. a type with exactly one possible value, where its value does not have a defined dynamic representation. + + #### Example: + + ```mlir + func.func @none_type() { + %none_val = "foo.unknown_op"() : () -> none + return + } + ``` }]; let extraClassDeclaration = [{ static NoneType get(MLIRContext *context); @@ -692,7 +717,7 @@ def Builtin_Opaque : Builtin_Type<"Opaque", "opaque"> { represented in their raw string form, and can only usefully be tested for type equality. - Examples: + #### Example: ```mlir opaque<"llvm", "struct<(i32, float)>"> @@ -761,7 +786,7 @@ def Builtin_RankedTensor : Builtin_Type<"RankedTensor", "tensor", [ zero sizes are not allowed in some other types, such tensors should be optimized away before lowering tensors to vectors. - Examples: + #### Example: ```mlir // Known rank but unknown dimensions. @@ -846,7 +871,7 @@ def Builtin_Tuple : Builtin_Type<"Tuple", "tuple"> { provides no standard operations for operating on `tuple` types ([rationale](../../Rationale/Rationale/#tuple-types)). - Examples: + #### Example: ```mlir // Empty tuple. @@ -917,7 +942,7 @@ def Builtin_UnrankedMemRef : Builtin_Type<"UnrankedMemRef", "unranked_memref", [ See [MemRefType](#memreftype) for more information on memref types. - Examples: + #### Examples: ```mlir memref<*f32> @@ -990,7 +1015,7 @@ def Builtin_UnrankedTensor : Builtin_Type<"UnrankedTensor", "unranked_tensor", [ unknown rank. See [RankedTensorType](#rankedtensortype) for more information on tensor types. - Examples: + #### Examples: ```mlir tensor<*xf32> @@ -1051,7 +1076,7 @@ def Builtin_Vector : Builtin_Type<"Vector", "vector", [ShapedTypeInterface], "Ty declarations, `vector<0x42xi32>` is invalid because it is interpreted as a 2D vector with shape `(0, 42)` and zero shapes are not allowed. - Examples: + #### Examples: ```mlir // A 2D fixed-length vector of 3x42 i32 elements. diff --git a/mlir/include/mlir/IR/OpImplementation.h b/mlir/include/mlir/IR/OpImplementation.h index f1fabf95a68b..86ed14e7ca84 100644 --- a/mlir/include/mlir/IR/OpImplementation.h +++ b/mlir/include/mlir/IR/OpImplementation.h @@ -226,6 +226,8 @@ public: printArrowTypeList(results); } + void printDimensionList(ArrayRef shape); + /// Class used to automatically end a cyclic region on destruction. class CyclicPrintReset { public: @@ -1762,6 +1764,17 @@ public: const SetVector &referencedResources, AsmResourceBuilder &builder) const {} }; + +//===--------------------------------------------------------------------===// +// Custom printers and parsers. +//===--------------------------------------------------------------------===// + +// Handles custom(...) in TableGen. +void printDimensionList(OpAsmPrinter &printer, Operation *op, + ArrayRef dimensions); +ParseResult parseDimensionList(OpAsmParser &parser, + DenseI64ArrayAttr &dimensions); + } // namespace mlir //===--------------------------------------------------------------------===// diff --git a/mlir/include/mlir/Pass/Pass.h b/mlir/include/mlir/Pass/Pass.h index 5a4df4324ecd..121b253eb83f 100644 --- a/mlir/include/mlir/Pass/Pass.h +++ b/mlir/include/mlir/Pass/Pass.h @@ -9,6 +9,7 @@ #ifndef MLIR_PASS_PASS_H #define MLIR_PASS_PASS_H +#include "mlir/IR/Action.h" #include "mlir/Pass/AnalysisManager.h" #include "mlir/Pass/PassRegistry.h" #include "mlir/Support/LogicalResult.h" @@ -457,6 +458,52 @@ protected: } }; +/// This class encapsulates the "action" of executing a single pass. This allows +/// a user of the Action infrastructure to query information about an action in +/// (for example) a breakpoint context. You could use it like this: +/// +/// auto onBreakpoint = [&](const ActionActiveStack *backtrace) { +/// if (auto passExec = dyn_cast(anAction)) +/// record(passExec.getPass()); +/// return ExecutionContext::Apply; +/// }; +/// ExecutionContext exeCtx(onBreakpoint); +/// +class PassExecutionAction : public tracing::ActionImpl { + using Base = tracing::ActionImpl; + +public: + /// Define a TypeID for this PassExecutionAction. + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(PassExecutionAction) + /// Construct a PassExecutionAction. This is called by the OpToOpPassAdaptor + /// when it calls `executeAction`. + PassExecutionAction(ArrayRef irUnits, const Pass &pass); + + /// The tag required by ActionImpl to identify this action. + static constexpr StringLiteral tag = "pass-execution"; + + /// Print a textual version of this action to `os`. + void print(raw_ostream &os) const override; + + /// Get the pass that will be executed by this action. This is not a class of + /// passes, or all instances of a pass kind, this is a single pass. + const Pass &getPass() const { return pass; } + + /// Get the operation that is the base of this pass. For example, an + /// OperationPass would return a ModuleOp. + Operation *getOp() const; + +public: + /// Reference to the pass being run. Notice that this will *not* extend the + /// lifetime of the pass, and so this class is therefore unsafe to keep past + /// the lifetime of the `executeAction` call. + const Pass &pass; + + /// The base op for this pass. For an OperationPass, we would have a + /// ModuleOp here. + Operation *op; +}; + } // namespace mlir #endif // MLIR_PASS_PASS_H diff --git a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h index b8e449dc11df..b49d2f539453 100644 --- a/mlir/include/mlir/Target/LLVMIR/ModuleImport.h +++ b/mlir/include/mlir/Target/LLVMIR/ModuleImport.h @@ -172,6 +172,12 @@ public: /// attributes of LLVMFuncOp `funcOp`. void processFunctionAttributes(llvm::Function *func, LLVMFuncOp funcOp); + /// Sets the integer overflow flags (nsw/nuw) attribute for the imported + /// operation `op` given the original instruction `inst`. Asserts if the + /// operation does not implement the integer overflow flag interface. + void setIntegerOverflowFlagsAttr(llvm::Instruction *inst, + Operation *op) const; + /// Sets the fastmath flags attribute for the imported operation `op` given /// the original instruction `inst`. Asserts if the operation does not /// implement the fastmath interface. diff --git a/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp b/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp index 39b3a0996396..a82c30717e27 100644 --- a/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp +++ b/mlir/lib/Analysis/DataFlow/IntegerRangeAnalysis.cpp @@ -180,7 +180,7 @@ void IntegerRangeAnalysis::visitNonControlFlowArguments( } else if (auto value = llvm::dyn_cast_if_present(*loopBound)) { const IntegerValueRangeLattice *lattice = getLatticeElementFor(op, value); - if (lattice != nullptr) + if (lattice != nullptr && !lattice->getValue().isUninitialized()) return getUpper ? lattice->getValue().getValue().smax() : lattice->getValue().getValue().smin(); } diff --git a/mlir/lib/Bindings/Python/ExecutionEngineModule.cpp b/mlir/lib/Bindings/Python/ExecutionEngineModule.cpp index 3f834259622f..b3df30583fc9 100644 --- a/mlir/lib/Bindings/Python/ExecutionEngineModule.cpp +++ b/mlir/lib/Bindings/Python/ExecutionEngineModule.cpp @@ -20,7 +20,7 @@ class PyExecutionEngine { public: PyExecutionEngine(MlirExecutionEngine executionEngine) : executionEngine(executionEngine) {} - PyExecutionEngine(PyExecutionEngine &&other) + PyExecutionEngine(PyExecutionEngine &&other) noexcept : executionEngine(other.executionEngine) { other.executionEngine.ptr = nullptr; } diff --git a/mlir/lib/Bindings/Python/IRInterfaces.cpp b/mlir/lib/Bindings/Python/IRInterfaces.cpp index c3aac0b092bc..54cfa56066eb 100644 --- a/mlir/lib/Bindings/Python/IRInterfaces.cpp +++ b/mlir/lib/Bindings/Python/IRInterfaces.cpp @@ -326,7 +326,7 @@ public: : shape(std::move(shape)), elementType(elementType), attribute(attribute), ranked(true) {} PyShapedTypeComponents(PyShapedTypeComponents &) = delete; - PyShapedTypeComponents(PyShapedTypeComponents &&other) + PyShapedTypeComponents(PyShapedTypeComponents &&other) noexcept : shape(other.shape), elementType(other.elementType), attribute(other.attribute), ranked(other.ranked) {} diff --git a/mlir/lib/Bindings/Python/IRModule.h b/mlir/lib/Bindings/Python/IRModule.h index d99b87d19bbe..79b7e0c96188 100644 --- a/mlir/lib/Bindings/Python/IRModule.h +++ b/mlir/lib/Bindings/Python/IRModule.h @@ -4,6 +4,7 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception //===----------------------------------------------------------------------===// #ifndef MLIR_BINDINGS_PYTHON_IRMODULES_H @@ -53,7 +54,7 @@ public: "cannot construct PyObjectRef with null referrent"); assert(this->object && "cannot construct PyObjectRef with null object"); } - PyObjectRef(PyObjectRef &&other) + PyObjectRef(PyObjectRef &&other) noexcept : referrent(other.referrent), object(std::move(other.object)) { other.referrent = nullptr; assert(!other.object); @@ -484,7 +485,8 @@ public: mlirDialectRegistryDestroy(registry); } PyDialectRegistry(PyDialectRegistry &) = delete; - PyDialectRegistry(PyDialectRegistry &&other) : registry(other.registry) { + PyDialectRegistry(PyDialectRegistry &&other) noexcept + : registry(other.registry) { other.registry = {nullptr}; } diff --git a/mlir/lib/Bindings/Python/Pass.cpp b/mlir/lib/Bindings/Python/Pass.cpp index 588a8e25414c..a68421b61641 100644 --- a/mlir/lib/Bindings/Python/Pass.cpp +++ b/mlir/lib/Bindings/Python/Pass.cpp @@ -23,7 +23,8 @@ namespace { class PyPassManager { public: PyPassManager(MlirPassManager passManager) : passManager(passManager) {} - PyPassManager(PyPassManager &&other) : passManager(other.passManager) { + PyPassManager(PyPassManager &&other) noexcept + : passManager(other.passManager) { other.passManager.ptr = nullptr; } ~PyPassManager() { diff --git a/mlir/lib/CAPI/IR/IR.cpp b/mlir/lib/CAPI/IR/IR.cpp index d1ee1b774c34..ac9889df11f8 100644 --- a/mlir/lib/CAPI/IR/IR.cpp +++ b/mlir/lib/CAPI/IR/IR.cpp @@ -986,6 +986,10 @@ MlirOperation mlirOpOperandGetOwner(MlirOpOperand opOperand) { return wrap(unwrap(opOperand)->getOwner()); } +MlirValue mlirOpOperandGetValue(MlirOpOperand opOperand) { + return wrap(unwrap(opOperand)->get()); +} + unsigned mlirOpOperandGetOperandNumber(MlirOpOperand opOperand) { return unwrap(opOperand)->getOperandNumber(); } diff --git a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp index c3c9780318a9..c9d7c0c313b5 100644 --- a/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp +++ b/mlir/lib/Conversion/ArmSMEToSCF/ArmSMEToSCF.cpp @@ -484,12 +484,6 @@ struct ConvertArmSMEToSCFPass target.addLegalDialect(); target.addIllegalOp(); - target.addDynamicallyLegalOp([](vector::PrintOp op) { - if (!op.getSource()) - return true; - VectorType vectorType = dyn_cast(op.getPrintType()); - return !vectorType || !arm_sme::isValidSMETileVectorType(vectorType); - }); if (failed(applyPartialConversion(getOperation(), target, std::move(patterns)))) signalPassFailure(); diff --git a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp index 75dee09d2f64..b68baffb5a85 100644 --- a/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp +++ b/mlir/lib/Conversion/GPUCommon/GPUToLLVMConversion.cpp @@ -1340,18 +1340,18 @@ LogicalResult ConvertSetDefaultDeviceOpToGpuRuntimeCallPattern::matchAndRewrite( } template -static Value genConstInt32From(OpBuilder &builder, Location loc, T TValue) { +static Value genConstInt32From(OpBuilder &builder, Location loc, T tValue) { Type llvmInt32Type = builder.getIntegerType(32); return builder.create(loc, llvmInt32Type, - static_cast(TValue)); + static_cast(tValue)); } template -static Value genConstFloat32From(OpBuilder &builder, Location loc, T TValue) { +static Value genConstFloat32From(OpBuilder &builder, Location loc, T tValue) { Type llvmFloat32Type = builder.getF32Type(); return builder.create( loc, llvmFloat32Type, - builder.getF32FloatAttr(static_cast(TValue))); + builder.getF32FloatAttr(static_cast(tValue))); } LogicalResult ConvertCreateDnTensorOpToGpuRuntimeCallPattern::matchAndRewrite( @@ -1629,7 +1629,7 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite( auto stream = adaptor.getAsyncDependencies().front(); Value bufferSize; if (is2To4Sparsity(op.getSpmatA())) { - auto prune_flag = + auto pruneFlag = genConstInt32From(rewriter, loc, get2To4PruneFlag(op.getSpmatA())); auto computeType = genConstInt32From( rewriter, loc, getCuSparseLtDataTypeFrom(adaptor.getComputeType())); @@ -1641,7 +1641,7 @@ LogicalResult ConvertSpMMBufferSizeOpToGpuRuntimeCallPattern::matchAndRewrite( .create(loc, rewriter, {bufferSize, modeA, modeB, adaptor.getSpmatA(), adaptor.getDnmatB(), adaptor.getDnmatC(), computeType, - prune_flag, stream}) + pruneFlag, stream}) .getResult(); auto bufferSizePtr1 = rewriter.create( diff --git a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp index 103c1fb8c382..27c2cb935207 100644 --- a/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp +++ b/mlir/lib/Conversion/MathToLibm/MathToLibm.cpp @@ -162,6 +162,7 @@ ScalarOpToLibmCall::matchAndRewrite(Op op, void mlir::populateMathToLibmConversionPatterns(RewritePatternSet &patterns) { MLIRContext *ctx = patterns.getContext(); + populatePatternsForOp(patterns, ctx, "acosf", "acos"); populatePatternsForOp(patterns, ctx, "atan2f", "atan2"); populatePatternsForOp(patterns, ctx, "atanf", "atan"); populatePatternsForOp(patterns, ctx, "cbrtf", "cbrt"); diff --git a/mlir/lib/Conversion/MemRefToSPIRV/MapMemRefStorageClassPass.cpp b/mlir/lib/Conversion/MemRefToSPIRV/MapMemRefStorageClassPass.cpp index 35a3af07940a..c6ef5be2494a 100644 --- a/mlir/lib/Conversion/MemRefToSPIRV/MapMemRefStorageClassPass.cpp +++ b/mlir/lib/Conversion/MemRefToSPIRV/MapMemRefStorageClassPass.cpp @@ -205,7 +205,7 @@ spirv::MemorySpaceToStorageClassConverter::MemorySpaceToStorageClassConverter( static bool isLegalType(Type type) { if (auto memRefType = dyn_cast(type)) { Attribute spaceAttr = memRefType.getMemorySpace(); - return spaceAttr && isa(spaceAttr); + return isa_and_nonnull(spaceAttr); } return true; } diff --git a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp index baab40020ac2..a9c3b0a71ef0 100644 --- a/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp +++ b/mlir/lib/Conversion/PDLToPDLInterp/PredicateTree.cpp @@ -856,7 +856,7 @@ static void foldSwitchToBool(std::unique_ptr &node) { // If the node only contains one child, collapse it into a boolean predicate // node. if (children.size() == 1) { - auto childIt = children.begin(); + auto *childIt = children.begin(); node = std::make_unique( node->getPosition(), node->getQuestion(), childIt->first, std::move(childIt->second), std::move(node->getFailureNode())); diff --git a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp index 429d1137b6f3..56fcf25c9ddc 100644 --- a/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp +++ b/mlir/lib/Conversion/VectorToGPU/VectorToGPU.cpp @@ -303,8 +303,9 @@ static bool supportsMMaMatrixType(Operation *op, bool useNvGpu) { /// `getSlice`. In scf.for we only want to include as part of the slice elements /// that are part of the use/def chain. static SetVector -getSliceContract(Operation *op, BackwardSliceOptions backwardSliceOptions, - ForwardSliceOptions forwardSliceOptions) { +getSliceContract(Operation *op, + const BackwardSliceOptions &backwardSliceOptions, + const ForwardSliceOptions &forwardSliceOptions) { SetVector slice; slice.insert(op); unsigned currentIndex = 0; @@ -455,7 +456,8 @@ struct CombineTransferReadOpTranspose final Type resultType = op.getType(); Operation *extOp; if ((extOp = source.getDefiningOp()) || - (extOp = source.getDefiningOp())) { + (extOp = source.getDefiningOp()) || + (extOp = source.getDefiningOp())) { source = extOp->getOperand(0); resultType = VectorType::get(cast(resultType).getShape(), @@ -493,9 +495,12 @@ struct CombineTransferReadOpTranspose final if (isa(extOp)) result = rewriter.create(loc, op.getType(), result) .getResult(); - else + else if (isa(extOp)) result = rewriter.create(loc, op.getType(), result) .getResult(); + else + result = rewriter.create(loc, op.getType(), result) + .getResult(); } rewriter.replaceOp(op, result); @@ -553,7 +558,7 @@ convertTransferReadOp(RewriterBase &rewriter, vector::TransferReadOp op, auto elType = op.getVectorType().getElementType(); const char *fragType = inferFragType(op); if (op->hasOneUse()) { - auto user = *op->user_begin(); + auto *user = *op->user_begin(); // Infer the signedness of the mma type from the integer extend. bool isSignedExtend = isa(user); if (isSignedExtend || isa(user)) { diff --git a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp index a7fc7ddec26e..7f2f3c3410c3 100644 --- a/mlir/lib/Dialect/Affine/IR/AffineOps.cpp +++ b/mlir/lib/Dialect/Affine/IR/AffineOps.cpp @@ -4474,6 +4474,17 @@ LogicalResult AffineVectorStoreOp::verify() { // DelinearizeIndexOp //===----------------------------------------------------------------------===// +LogicalResult AffineDelinearizeIndexOp::inferReturnTypes( + MLIRContext *context, std::optional<::mlir::Location> location, + ValueRange operands, DictionaryAttr attributes, OpaqueProperties properties, + RegionRange regions, SmallVectorImpl &inferredReturnTypes) { + AffineDelinearizeIndexOpAdaptor adaptor(operands, attributes, properties, + regions); + inferredReturnTypes.assign(adaptor.getBasis().size(), + IndexType::get(context)); + return success(); +} + void AffineDelinearizeIndexOp::build(OpBuilder &builder, OperationState &result, Value linearIndex, ArrayRef basis) { diff --git a/mlir/lib/Dialect/Affine/IR/CMakeLists.txt b/mlir/lib/Dialect/Affine/IR/CMakeLists.txt index 9e3c1161fd92..7f7a01be891e 100644 --- a/mlir/lib/Dialect/Affine/IR/CMakeLists.txt +++ b/mlir/lib/Dialect/Affine/IR/CMakeLists.txt @@ -15,6 +15,7 @@ add_mlir_dialect_library(MLIRAffineDialect MLIRArithDialect MLIRDialectUtils MLIRIR + MLIRInferTypeOpInterface MLIRLoopLikeInterface MLIRMemRefDialect MLIRShapedOpInterfaces diff --git a/mlir/lib/Dialect/Affine/Utils/CMakeLists.txt b/mlir/lib/Dialect/Affine/Utils/CMakeLists.txt index c2ce43f4eebb..ef6e0dbf45d3 100644 --- a/mlir/lib/Dialect/Affine/Utils/CMakeLists.txt +++ b/mlir/lib/Dialect/Affine/Utils/CMakeLists.txt @@ -12,6 +12,7 @@ add_mlir_dialect_library(MLIRAffineUtils MLIRAffineAnalysis MLIRAnalysis MLIRArithUtils + MLIRFuncDialect MLIRMemRefDialect MLIRTransformUtils MLIRViewLikeInterface diff --git a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp index 6d8706775758..5529dccaf1de 100644 --- a/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp +++ b/mlir/lib/Dialect/Complex/IR/ComplexOps.cpp @@ -58,10 +58,12 @@ LogicalResult ConstantOp::verify() { } auto complexEltTy = getType().getElementType(); - auto re = llvm::dyn_cast(arrayAttr[0]); - auto im = llvm::dyn_cast(arrayAttr[1]); - if (!re || !im) - return emitOpError("requires attribute's elements to be float attributes"); + if (!isa(arrayAttr[0]) || + !isa(arrayAttr[1])) + return emitOpError( + "requires attribute's elements to be float or integer attributes"); + auto re = llvm::dyn_cast(arrayAttr[0]); + auto im = llvm::dyn_cast(arrayAttr[1]); if (complexEltTy != re.getType() || complexEltTy != im.getType()) { return emitOpError() << "requires attribute's element types (" << re.getType() << ", " diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp index c67bba04d697..53e1088f620d 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp @@ -69,7 +69,13 @@ static ParseResult parseLLVMOpAttrs(OpAsmParser &parser, static void printLLVMOpAttrs(OpAsmPrinter &printer, Operation *op, DictionaryAttr attrs) { - printer.printOptionalAttrDict(processFMFAttr(attrs.getValue())); + auto filteredAttrs = processFMFAttr(attrs.getValue()); + if (auto iface = dyn_cast(op)) + printer.printOptionalAttrDict( + filteredAttrs, + /*elidedAttrs=*/{iface.getIntegerOverflowAttrName()}); + else + printer.printOptionalAttrDict(filteredAttrs); } /// Verifies `symbol`'s use in `op` to ensure the symbol is a valid and diff --git a/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp b/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp index b094c650ff19..cf900ac0be8f 100644 --- a/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp +++ b/mlir/lib/Dialect/LLVMIR/Transforms/TypeConsistency.cpp @@ -161,7 +161,10 @@ static std::optional gepToByteOffset(DataLayout &layout, GEPOp gep) { IntegerAttr indexInt = llvm::dyn_cast_if_present(index); if (!indexInt) return std::nullopt; - indices.push_back(indexInt.getInt()); + int32_t gepIndex = indexInt.getInt(); + if (gepIndex < 0) + return std::nullopt; + indices.push_back(static_cast(gepIndex)); } uint64_t offset = indices[0] * layout.getTypeSize(gep.getElemType()); diff --git a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp index e3713457e841..54055aefbc51 100644 --- a/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp +++ b/mlir/lib/Dialect/Linalg/TransformOps/LinalgTransformOps.cpp @@ -1171,6 +1171,7 @@ transform::MatchOp::apply(transform::TransformRewriter &rewriter, } SmallVector res; + bool incorrectNumOperandTypes = false; auto matchFun = [&](Operation *op) { if (getOps().has_value() && !strs.contains(op->getName().getStringRef())) return; @@ -1210,12 +1211,47 @@ transform::MatchOp::apply(transform::TransformRewriter &rewriter, return; } + if (getFilterOperandTypes().has_value()) { + mlir::ArrayAttr types = getFilterOperandTypes().value(); + auto operandTypes = op->getOperandTypes(); + + if (types.size() == 1) { + // All the operands must must be equal to the specified type + auto typeattr = + dyn_cast(getFilterOperandTypes().value()[0]); + Type t = typeattr.getValue().cast<::mlir::Type>(); + if (!llvm::all_of(op->getOperandTypes(), + [&](Type operandType) { return operandType == t; })) + return; + } else { + // The operand types must match all the types in the list (in the same + // order in with they are specified) + if (types.size() != operandTypes.size()) { + incorrectNumOperandTypes = true; + return; + } + + for (auto [attr, operandType] : + llvm::zip_equal(getFilterOperandTypes().value(), operandTypes)) { + auto typeattr = cast(attr); + Type type = typeattr.getValue().cast<::mlir::Type>(); + + if (type != operandType) + return; + } + } + } + // All constraints are satisfied. res.push_back(op); return; }; (*payloadOps.begin())->walk(matchFun); + if (incorrectNumOperandTypes) + return emitDefiniteFailure("If filter_operand_types contains more than a " + "type, then it must contain as much types as " + "the number of operands in the target ops"); results.set(cast(getResult()), res); return DiagnosedSilenceableFailure::success(); } diff --git a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp index f0393e44fc00..3eb91190751e 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/ElementwiseOpFusion.cpp @@ -71,6 +71,25 @@ static AffineMap getIndexingMapOfProducerOperandsInCoordinatesOfFusedOp( return t1.compose(fusedConsumerArgIndexMap); } +/// Returns a set of indices of the producer's results which would +/// be preserved after the fusion. +llvm::SmallDenseSet +ElementwiseOpFusionResult::getPreservedProducerResults(GenericOp producer, + GenericOp consumer) { + llvm::SmallDenseSet preservedProducerResults; + for (const auto &producerResult : llvm::enumerate(producer->getResults())) { + auto *outputOperand = producer.getDpsInitOperand(producerResult.index()); + if (producer.payloadUsesValueFromOperand(outputOperand) || + !producer.canOpOperandsBeDropped(outputOperand) || + llvm::any_of(producerResult.value().getUsers(), [&](Operation *user) { + return user != consumer.getOperation(); + })) { + preservedProducerResults.insert(producerResult.index()); + } + } + return preservedProducerResults; +} + /// Conditions for elementwise fusion of generic operations. bool mlir::linalg::areElementwiseOpsFusable(OpOperand *fusedOperand) { if (!fusedOperand) @@ -285,17 +304,9 @@ mlir::linalg::fuseElementwiseOps(RewriterBase &rewriter, assert(consumer.isDpsInput(fusedOperand) && "expected producer of input operand"); /// Find the results of the producer that have uses outside of the consumer. - llvm::SmallDenseSet preservedProducerResults; - for (const auto &producerResult : llvm::enumerate(producer->getResults())) { - auto *outputOperand = producer.getDpsInitOperand(producerResult.index()); - if (producer.payloadUsesValueFromOperand(outputOperand) || - !producer.canOpOperandsBeDropped(outputOperand) || - llvm::any_of(producerResult.value().getUsers(), [&](Operation *user) { - return user != consumer.getOperation(); - })) { - preservedProducerResults.insert(producerResult.index()); - } - } + llvm::SmallDenseSet preservedProducerResults = + ElementwiseOpFusionResult::getPreservedProducerResults(producer, + consumer); // Compute the fused operands list and indexing maps. SmallVector fusedInputOperands, fusedOutputOperands; @@ -422,11 +433,6 @@ public: if (!controlFn(&opOperand)) continue; - // Find the producer of the operand. - FailureOr fusionResult = - fuseElementwiseOps(rewriter, &opOperand); - if (failed(fusionResult)) - return rewriter.notifyMatchFailure(genericOp, "fusion failed"); Operation *producer = opOperand.get().getDefiningOp(); // Do not fuse a sparse-in/dense-out operation, as the @@ -435,6 +441,12 @@ public: !sparse_tensor::hasAnySparseResult(producer)) return failure(); + // Find the producer of the operand. + FailureOr fusionResult = + fuseElementwiseOps(rewriter, &opOperand); + if (failed(fusionResult)) + return rewriter.notifyMatchFailure(genericOp, "fusion failed"); + // Perform the fusion. for (auto [origVal, replacement] : fusionResult->replacements) { rewriter.replaceUsesWithIf(origVal, replacement, [&](OpOperand &use) { diff --git a/mlir/lib/Dialect/Math/IR/MathOps.cpp b/mlir/lib/Dialect/Math/IR/MathOps.cpp index 28d1c062f235..066a21c76f7d 100644 --- a/mlir/lib/Dialect/Math/IR/MathOps.cpp +++ b/mlir/lib/Dialect/Math/IR/MathOps.cpp @@ -41,6 +41,24 @@ OpFoldResult math::AbsIOp::fold(FoldAdaptor adaptor) { [](const APInt &a) { return a.abs(); }); } +//===----------------------------------------------------------------------===// +// AcosOp folder +//===----------------------------------------------------------------------===// + +OpFoldResult math::AcosOp::fold(FoldAdaptor adaptor) { + return constFoldUnaryOpConditional( + adaptor.getOperands(), [](const APFloat &a) -> std::optional { + switch (a.getSizeInBits(a.getSemantics())) { + case 64: + return APFloat(acos(a.convertToDouble())); + case 32: + return APFloat(acosf(a.convertToFloat())); + default: + return {}; + } + }); +} + //===----------------------------------------------------------------------===// // AtanOp folder //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp index 3b89860c14d9..1ba95f21ec7f 100644 --- a/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp +++ b/mlir/lib/Dialect/Mesh/IR/MeshOps.cpp @@ -58,11 +58,6 @@ static SmallVector &canonicalizeSetAsVector(SmallVector &vec) { return vec; } -template -static bool isMeshDimensionDynamic(DimSize size) { - return size <= DimSize(0); -} - using MeshAxis = int16_t; namespace { @@ -161,9 +156,9 @@ LogicalResult ClusterOp::verify() { "rank of dim_sizes is not expected to be larger than rank of cluster"); for (int64_t dimSize : dimSizes) { - if (dimSize < 0) - return emitOpError( - "dimension size of a mesh cluster is expected to be non-negative"); + if (dimSize < 0 && !ShapedType::isDynamic(dimSize)) + return emitOpError("dimension size of a mesh cluster is expected to be " + "non-negative or dynamic"); } return success(); @@ -316,7 +311,7 @@ static int64_t collectiveDeviceGroupSize(ArrayRef meshAxes, int64_t res = 1; for (MeshAxis axis : meshAxes) { - if (isMeshDimensionDynamic(meshShape[axis])) { + if (ShapedType::isDynamic(meshShape[axis])) { return ShapedType::kDynamic; } assert(size_t(axis) < meshShape.size()); diff --git a/mlir/lib/Dialect/SCF/IR/CMakeLists.txt b/mlir/lib/Dialect/SCF/IR/CMakeLists.txt index 9882b843c285..423e1c3e1e04 100644 --- a/mlir/lib/Dialect/SCF/IR/CMakeLists.txt +++ b/mlir/lib/Dialect/SCF/IR/CMakeLists.txt @@ -11,12 +11,13 @@ add_mlir_dialect_library(MLIRSCFDialect LINK_LIBS PUBLIC MLIRArithDialect - MLIRBufferizationDialect MLIRControlFlowDialect + MLIRDialectUtils MLIRFunctionInterfaces MLIRIR MLIRLoopLikeInterface MLIRSideEffectInterfaces + MLIRTensorDialect MLIRValueBoundsOpInterface ) diff --git a/mlir/lib/Dialect/SCF/IR/SCF.cpp b/mlir/lib/Dialect/SCF/IR/SCF.cpp index 2e7382160f8e..55bb5788108b 100644 --- a/mlir/lib/Dialect/SCF/IR/SCF.cpp +++ b/mlir/lib/Dialect/SCF/IR/SCF.cpp @@ -9,7 +9,6 @@ #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/Dialect/Arith/IR/Arith.h" #include "mlir/Dialect/Arith/Utils/Utils.h" -#include "mlir/Dialect/Bufferization/IR/Bufferization.h" #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/Dialect/SCF/IR/DeviceMappingInterface.h" @@ -1082,139 +1081,12 @@ struct ForOpTensorCastFolder : public OpRewritePattern { } }; -/// Canonicalize the iter_args of an scf::ForOp that involve a -/// `bufferization.to_tensor` and for which only the last loop iteration is -/// actually visible outside of the loop. The canonicalization looks for a -/// pattern such as: -/// ``` -/// %t0 = ... : tensor_type -/// %0 = scf.for ... iter_args(%bb0 : %t0) -> (tensor_type) { -/// ... -/// // %m is either buffer_cast(%bb00) or defined above the loop -/// %m... : memref_type -/// ... // uses of %m with potential inplace updates -/// %new_tensor = bufferization.to_tensor %m : memref_type -/// ... -/// scf.yield %new_tensor : tensor_type -/// } -/// ``` -/// -/// `%bb0` may have either 0 or 1 use. If it has 1 use it must be exactly a -/// `%m = buffer_cast %bb0` op that feeds into the yielded -/// `bufferization.to_tensor` op. -/// -/// If no aliasing write to the memref `%m`, from which `%new_tensor`is loaded, -/// occurs between `bufferization.to_tensor and yield then the value %0 -/// visible outside of the loop is the last `bufferization.to_tensor` -/// produced in the loop. -/// -/// For now, we approximate the absence of aliasing by only supporting the case -/// when the bufferization.to_tensor is the operation immediately preceding -/// the yield. -// -/// The canonicalization rewrites the pattern as: -/// ``` -/// // %m is either a buffer_cast or defined above -/// %m... : memref_type -/// scf.for ... iter_args(%bb0 : %t0) -> (tensor_type) { -/// ... // uses of %m with potential inplace updates -/// scf.yield %bb0: tensor_type -/// } -/// %0 = bufferization.to_tensor %m : memref_type -/// ``` -/// -/// A later bbArg canonicalization will further rewrite as: -/// ``` -/// // %m is either a buffer_cast or defined above -/// %m... : memref_type -/// scf.for ... { // no iter_args -/// ... // uses of %m with potential inplace updates -/// } -/// %0 = bufferization.to_tensor %m : memref_type -/// ``` -struct LastTensorLoadCanonicalization : public OpRewritePattern { - using OpRewritePattern::OpRewritePattern; - - LogicalResult matchAndRewrite(ForOp forOp, - PatternRewriter &rewriter) const override { - assert(std::next(forOp.getRegion().begin()) == forOp.getRegion().end() && - "unexpected multiple blocks"); - - Location loc = forOp.getLoc(); - DenseMap replacements; - for (BlockArgument bbArg : forOp.getRegionIterArgs()) { - unsigned idx = bbArg.getArgNumber() - /*numIv=*/1; - auto yieldOp = - cast(forOp.getRegion().front().getTerminator()); - Value yieldVal = yieldOp->getOperand(idx); - auto tensorLoadOp = yieldVal.getDefiningOp(); - bool isTensor = llvm::isa(bbArg.getType()); - - bufferization::ToMemrefOp tensorToMemref; - // Either bbArg has no use or it has a single buffer_cast use. - if (bbArg.hasOneUse()) - tensorToMemref = - dyn_cast(*bbArg.getUsers().begin()); - if (!isTensor || !tensorLoadOp || (!bbArg.use_empty() && !tensorToMemref)) - continue; - // If tensorToMemref is present, it must feed into the `ToTensorOp`. - if (tensorToMemref && tensorLoadOp.getMemref() != tensorToMemref) - continue; - // TODO: Any aliasing write of tensorLoadOp.memref() nested under `forOp` - // must be before `ToTensorOp` in the block so that the lastWrite - // property is not subject to additional side-effects. - // For now, we only support the case when ToTensorOp appears - // immediately before the terminator. - if (tensorLoadOp->getNextNode() != yieldOp) - continue; - - // Clone the optional tensorToMemref before forOp. - if (tensorToMemref) { - rewriter.setInsertionPoint(forOp); - rewriter.replaceOpWithNewOp( - tensorToMemref, tensorToMemref.getMemref().getType(), - tensorToMemref.getTensor()); - } - - // Clone the tensorLoad after forOp. - rewriter.setInsertionPointAfter(forOp); - Value newTensorLoad = rewriter.create( - loc, tensorLoadOp.getMemref()); - Value forOpResult = forOp.getResult(bbArg.getArgNumber() - /*iv=*/1); - replacements.insert(std::make_pair(forOpResult, newTensorLoad)); - - // Make the terminator just yield the bbArg, the old tensorLoadOp + the - // old bbArg (that is now directly yielded) will canonicalize away. - rewriter.startRootUpdate(yieldOp); - yieldOp.setOperand(idx, bbArg); - rewriter.finalizeRootUpdate(yieldOp); - } - if (replacements.empty()) - return failure(); - - // We want to replace a subset of the results of `forOp`. rewriter.replaceOp - // replaces the whole op and erase it unconditionally. This is wrong for - // `forOp` as it generally contains ops with side effects. - // Instead, use `rewriter.replaceOpWithIf`. - SmallVector newResults; - newResults.reserve(forOp.getNumResults()); - for (Value v : forOp.getResults()) { - auto it = replacements.find(v); - newResults.push_back((it != replacements.end()) ? it->second : v); - } - unsigned idx = 0; - rewriter.replaceOpWithIf(forOp, newResults, [&](OpOperand &op) { - return op.get() != newResults[idx++]; - }); - return success(); - } -}; } // namespace void ForOp::getCanonicalizationPatterns(RewritePatternSet &results, MLIRContext *context) { - results.add(context); + results.add( + context); } std::optional ForOp::getConstantStep() { diff --git a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp index 20fa8089201a..6c36600975a5 100644 --- a/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp +++ b/mlir/lib/Dialect/SCF/Transforms/LoopPipelining.cpp @@ -44,9 +44,10 @@ protected: unsigned maxStage = 0; DenseMap stages; std::vector opOrder; - int64_t ub; - int64_t lb; - int64_t step; + Value ub; + Value lb; + Value step; + bool dynamicLoop; PipeliningOption::AnnotationlFnType annotateFn = nullptr; bool peelEpilogue; PipeliningOption::PredicateOpFn predicateFn = nullptr; @@ -96,25 +97,41 @@ bool LoopPipelinerInternal::initializeLoopInfo( ForOp op, const PipeliningOption &options) { LDBG("Start initializeLoopInfo"); forOp = op; - auto upperBoundCst = - forOp.getUpperBound().getDefiningOp(); - auto lowerBoundCst = - forOp.getLowerBound().getDefiningOp(); - auto stepCst = forOp.getStep().getDefiningOp(); + ub = forOp.getUpperBound(); + lb = forOp.getLowerBound(); + step = forOp.getStep(); + + dynamicLoop = true; + auto upperBoundCst = getConstantIntValue(ub); + auto lowerBoundCst = getConstantIntValue(lb); + auto stepCst = getConstantIntValue(step); if (!upperBoundCst || !lowerBoundCst || !stepCst) { - LDBG("--no constant bounds or step -> BAIL"); - return false; + if (!options.supportDynamicLoops) { + LDBG("--dynamic loop not supported -> BAIL"); + return false; + } + } else { + int64_t ubImm = upperBoundCst.value(); + int64_t lbImm = lowerBoundCst.value(); + int64_t stepImm = stepCst.value(); + int64_t numIteration = ceilDiv(ubImm - lbImm, stepImm); + if (numIteration > maxStage) { + dynamicLoop = false; + } else if (!options.supportDynamicLoops) { + LDBG("--fewer loop iterations than pipeline stages -> BAIL"); + return false; + } } - ub = upperBoundCst.value(); - lb = lowerBoundCst.value(); - step = stepCst.value(); peelEpilogue = options.peelEpilogue; predicateFn = options.predicateFn; - if (!peelEpilogue && predicateFn == nullptr) { + if ((!peelEpilogue || dynamicLoop) && predicateFn == nullptr) { LDBG("--no epilogue or predicate set -> BAIL"); return false; } - int64_t numIteration = ceilDiv(ub - lb, step); + if (dynamicLoop && peelEpilogue) { + LDBG("--dynamic loop doesn't support epilogue yet -> BAIL"); + return false; + } std::vector> schedule; options.getScheduleFn(forOp, schedule); if (schedule.empty()) { @@ -128,10 +145,6 @@ bool LoopPipelinerInternal::initializeLoopInfo( stages[opSchedule.first] = opSchedule.second; opOrder.push_back(opSchedule.first); } - if (numIteration <= maxStage) { - LDBG("--fewer loop iterations than pipeline stages -> BAIL"); - return false; - } // All operations need to have a stage. for (Operation &op : forOp.getBody()->without_terminator()) { @@ -204,10 +217,31 @@ void LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) { setValueMapping(arg, operand.get(), 0); } auto yield = cast(forOp.getBody()->getTerminator()); + Location loc = forOp.getLoc(); + SmallVector predicates(maxStage); for (int64_t i = 0; i < maxStage; i++) { + if (dynamicLoop) { + Type t = ub.getType(); + // pred = ub > lb + (i * step) + Value iv = rewriter.create( + loc, lb, + rewriter.create( + loc, step, + rewriter.create( + loc, rewriter.getIntegerAttr(t, i)))); + predicates[i] = rewriter.create( + loc, arith::CmpIPredicate::slt, iv, ub); + } + // special handling for induction variable as the increment is implicit. - Value iv = - rewriter.create(forOp.getLoc(), lb + i * step); + // iv = lb + i * step + Type t = lb.getType(); + Value iv = rewriter.create( + loc, lb, + rewriter.create( + loc, step, + rewriter.create(loc, + rewriter.getIntegerAttr(t, i)))); setValueMapping(forOp.getInductionVar(), iv, i); for (Operation *op : opOrder) { if (stages[op] > i) @@ -220,6 +254,12 @@ void LoopPipelinerInternal::emitPrologue(RewriterBase &rewriter) { newOperand->set(replacement); } }); + int predicateIdx = i - stages[op]; + if (predicates[predicateIdx]) { + newOp = predicateFn(rewriter, newOp, predicates[predicateIdx]); + assert(newOp && "failed to predicate op."); + } + rewriter.setInsertionPointAfter(newOp); if (annotateFn) annotateFn(newOp, PipeliningOption::PipelinerPart::Prologue, i); for (unsigned destId : llvm::seq(unsigned(0), op->getNumResults())) { @@ -326,9 +366,16 @@ scf::ForOp LoopPipelinerInternal::createKernelLoop( // `numStages - 1` iterations. Then we adjust the upper bound to remove those // iterations. Value newUb = forOp.getUpperBound(); - if (peelEpilogue) - newUb = rewriter.create(forOp.getLoc(), - ub - maxStage * step); + if (peelEpilogue) { + Type t = ub.getType(); + Location loc = forOp.getLoc(); + // newUb = ub - maxStage * step + Value maxStageValue = rewriter.create( + loc, rewriter.getIntegerAttr(t, maxStage)); + Value maxStageByStep = + rewriter.create(loc, step, maxStageValue); + newUb = rewriter.create(loc, ub, maxStageByStep); + } auto newForOp = rewriter.create(forOp.getLoc(), forOp.getLowerBound(), newUb, forOp.getStep(), newLoopArg); @@ -358,9 +405,17 @@ LogicalResult LoopPipelinerInternal::createKernel( SmallVector predicates(maxStage + 1, nullptr); if (!peelEpilogue) { // Create a predicate for each stage except the last stage. + Location loc = newForOp.getLoc(); + Type t = ub.getType(); for (unsigned i = 0; i < maxStage; i++) { - Value c = rewriter.create( - newForOp.getLoc(), ub - (maxStage - i) * step); + // c = ub - (maxStage - i) * step + Value c = rewriter.create( + loc, ub, + rewriter.create( + loc, step, + rewriter.create( + loc, rewriter.getIntegerAttr(t, int64_t(maxStage - i))))); + Value pred = rewriter.create( newForOp.getLoc(), arith::CmpIPredicate::slt, newForOp.getInductionVar(), c); @@ -383,8 +438,14 @@ LogicalResult LoopPipelinerInternal::createKernel( // version incremented based on the stage where it is used. if (operand->get() == forOp.getInductionVar()) { rewriter.setInsertionPoint(newOp); - Value offset = rewriter.create( - forOp.getLoc(), (maxStage - stages[op]) * step); + + // offset = (maxStage - stages[op]) * step + Type t = step.getType(); + Value offset = rewriter.create( + forOp.getLoc(), step, + rewriter.create( + forOp.getLoc(), + rewriter.getIntegerAttr(t, maxStage - stages[op]))); Value iv = rewriter.create( forOp.getLoc(), newForOp.getInductionVar(), offset); nestedNewOp->setOperand(operand->getOperandNumber(), iv); @@ -508,8 +569,24 @@ LoopPipelinerInternal::emitEpilogue(RewriterBase &rewriter) { // Emit different versions of the induction variable. They will be // removed by dead code if not used. for (int64_t i = 0; i < maxStage; i++) { - Value newlastIter = rewriter.create( - forOp.getLoc(), lb + step * ((((ub - 1) - lb) / step) - i)); + Location loc = forOp.getLoc(); + Type t = lb.getType(); + Value minusOne = + rewriter.create(loc, rewriter.getIntegerAttr(t, -1)); + // number of iterations = ((ub - 1) - lb) / step + Value totalNumIteration = rewriter.create( + loc, + rewriter.create( + loc, rewriter.create(loc, ub, minusOne), lb), + step); + // newLastIter = lb + step * ((((ub - 1) - lb) / step) - i) + Value minusI = + rewriter.create(loc, rewriter.getIntegerAttr(t, -i)); + Value newlastIter = rewriter.create( + loc, lb, + rewriter.create( + loc, step, + rewriter.create(loc, totalNumIteration, minusI))); setValueMapping(forOp.getInductionVar(), newlastIter, maxStage - i); } // Emit `maxStage - 1` epilogue part that includes operations from stages diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp index 22cb9bf718e3..9de1707dfca4 100644 --- a/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp +++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVCanonicalization.cpp @@ -709,6 +709,195 @@ OpFoldResult spirv::LogicalOrOp::fold(FoldAdaptor adaptor) { return Attribute(); } +//===----------------------------------------------------------------------===// +// spirv.ShiftLeftLogical +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::ShiftLeftLogicalOp::fold( + spirv::ShiftLeftLogicalOp::FoldAdaptor adaptor) { + // x << 0 -> x + if (matchPattern(adaptor.getOperand2(), m_Zero())) { + return getOperand1(); + } + + // Unfortunately due to below undefined behaviour can't fold 0 for Base. + + // Results are computed per component, and within each component, per bit... + // + // The result is undefined if Shift is greater than or equal to the bit width + // of the components of Base. + // + // So we can use the APInt << method, but don't fold if undefined behaviour. + bool shiftToLarge = false; + auto res = constFoldBinaryOp( + adaptor.getOperands(), [&](const APInt &a, const APInt &b) { + if (shiftToLarge || b.uge(a.getBitWidth())) { + shiftToLarge = true; + return a; + } + return a << b; + }); + return shiftToLarge ? Attribute() : res; +} + +//===----------------------------------------------------------------------===// +// spirv.ShiftRightArithmetic +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::ShiftRightArithmeticOp::fold( + spirv::ShiftRightArithmeticOp::FoldAdaptor adaptor) { + // x >> 0 -> x + if (matchPattern(adaptor.getOperand2(), m_Zero())) { + return getOperand1(); + } + + // Unfortunately due to below undefined behaviour can't fold 0, -1 for Base. + + // Results are computed per component, and within each component, per bit... + // + // The result is undefined if Shift is greater than or equal to the bit width + // of the components of Base. + // + // So we can use the APInt ashr method, but don't fold if undefined behaviour. + bool shiftToLarge = false; + auto res = constFoldBinaryOp( + adaptor.getOperands(), [&](const APInt &a, const APInt &b) { + if (shiftToLarge || b.uge(a.getBitWidth())) { + shiftToLarge = true; + return a; + } + return a.ashr(b); + }); + return shiftToLarge ? Attribute() : res; +} + +//===----------------------------------------------------------------------===// +// spirv.ShiftRightLogical +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::ShiftRightLogicalOp::fold( + spirv::ShiftRightLogicalOp::FoldAdaptor adaptor) { + // x >> 0 -> x + if (matchPattern(adaptor.getOperand2(), m_Zero())) { + return getOperand1(); + } + + // Unfortunately due to below undefined behaviour can't fold 0 for Base. + + // Results are computed per component, and within each component, per bit... + // + // The result is undefined if Shift is greater than or equal to the bit width + // of the components of Base. + // + // So we can use the APInt lshr method, but don't fold if undefined behaviour. + bool shiftToLarge = false; + auto res = constFoldBinaryOp( + adaptor.getOperands(), [&](const APInt &a, const APInt &b) { + if (shiftToLarge || b.uge(a.getBitWidth())) { + shiftToLarge = true; + return a; + } + return a.lshr(b); + }); + return shiftToLarge ? Attribute() : res; +} + +//===----------------------------------------------------------------------===// +// spirv.BitwiseAndOp +//===----------------------------------------------------------------------===// + +OpFoldResult +spirv::BitwiseAndOp::fold(spirv::BitwiseAndOp::FoldAdaptor adaptor) { + // x & x -> x + if (getOperand1() == getOperand2()) { + return getOperand1(); + } + + APInt rhsMask; + if (matchPattern(adaptor.getOperand2(), m_ConstantInt(&rhsMask))) { + // x & 0 -> 0 + if (rhsMask.isZero()) + return getOperand2(); + + // x & -> x + if (rhsMask.isAllOnes()) + return getOperand1(); + + // (UConvert x : iN to iK) & -> UConvert x + if (auto zext = getOperand1().getDefiningOp()) { + int valueBits = + getElementTypeOrSelf(zext.getOperand()).getIntOrFloatBitWidth(); + if (rhsMask.zextOrTrunc(valueBits).isAllOnes()) + return getOperand1(); + } + } + + // According to the SPIR-V spec: + // + // Type is a scalar or vector of integer type. + // Results are computed per component, and within each component, per bit. + // So we can use the APInt & method. + return constFoldBinaryOp( + adaptor.getOperands(), + [](const APInt &a, const APInt &b) { return a & b; }); +} + +//===----------------------------------------------------------------------===// +// spirv.BitwiseOrOp +//===----------------------------------------------------------------------===// + +OpFoldResult spirv::BitwiseOrOp::fold(spirv::BitwiseOrOp::FoldAdaptor adaptor) { + // x | x -> x + if (getOperand1() == getOperand2()) { + return getOperand1(); + } + + APInt rhsMask; + if (matchPattern(adaptor.getOperand2(), m_ConstantInt(&rhsMask))) { + // x | 0 -> x + if (rhsMask.isZero()) + return getOperand1(); + + // x | -> + if (rhsMask.isAllOnes()) + return getOperand2(); + } + + // According to the SPIR-V spec: + // + // Type is a scalar or vector of integer type. + // Results are computed per component, and within each component, per bit. + // So we can use the APInt | method. + return constFoldBinaryOp( + adaptor.getOperands(), + [](const APInt &a, const APInt &b) { return a | b; }); +} + +//===----------------------------------------------------------------------===// +// spirv.BitwiseXorOp +//===----------------------------------------------------------------------===// + +OpFoldResult +spirv::BitwiseXorOp::fold(spirv::BitwiseXorOp::FoldAdaptor adaptor) { + // x ^ 0 -> x + if (matchPattern(adaptor.getOperand2(), m_Zero())) { + return getOperand1(); + } + + // x ^ x -> 0 + if (getOperand1() == getOperand2()) + return Builder(getContext()).getZeroAttr(getType()); + + // According to the SPIR-V spec: + // + // Type is a scalar or vector of integer type. + // Results are computed per component, and within each component, per bit. + // So we can use the APInt ^ method. + return constFoldBinaryOp( + adaptor.getOperands(), + [](const APInt &a, const APInt &b) { return a ^ b; }); +} + //===----------------------------------------------------------------------===// // spirv.mlir.selection //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp index 3906bf74ea72..2a1d08330828 100644 --- a/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp +++ b/mlir/lib/Dialect/SPIRV/IR/SPIRVOps.cpp @@ -1968,55 +1968,6 @@ LogicalResult spirv::ShiftRightLogicalOp::verify() { return verifyShiftOp(*this); } -//===----------------------------------------------------------------------===// -// spirv.BtiwiseAndOp -//===----------------------------------------------------------------------===// - -OpFoldResult -spirv::BitwiseAndOp::fold(spirv::BitwiseAndOp::FoldAdaptor adaptor) { - APInt rhsMask; - if (!matchPattern(adaptor.getOperand2(), m_ConstantInt(&rhsMask))) - return {}; - - // x & 0 -> 0 - if (rhsMask.isZero()) - return getOperand2(); - - // x & -> x - if (rhsMask.isAllOnes()) - return getOperand1(); - - // (UConvert x : iN to iK) & -> UConvert x - if (auto zext = getOperand1().getDefiningOp()) { - int valueBits = - getElementTypeOrSelf(zext.getOperand()).getIntOrFloatBitWidth(); - if (rhsMask.zextOrTrunc(valueBits).isAllOnes()) - return getOperand1(); - } - - return {}; -} - -//===----------------------------------------------------------------------===// -// spirv.BtiwiseOrOp -//===----------------------------------------------------------------------===// - -OpFoldResult spirv::BitwiseOrOp::fold(spirv::BitwiseOrOp::FoldAdaptor adaptor) { - APInt rhsMask; - if (!matchPattern(adaptor.getOperand2(), m_ConstantInt(&rhsMask))) - return {}; - - // x | 0 -> x - if (rhsMask.isZero()) - return getOperand1(); - - // x | -> - if (rhsMask.isAllOnes()) - return getOperand2(); - - return {}; -} - //===----------------------------------------------------------------------===// // spirv.ImageQuerySize //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp index 577dfe5ab2f3..686180c09da7 100644 --- a/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp +++ b/mlir/lib/Dialect/SparseTensor/IR/SparseTensorDialect.cpp @@ -858,22 +858,26 @@ bool mlir::sparse_tensor::isBlockSparsity(AffineMap dimToLvl) { std::map coeffientMap; for (auto result : dimToLvl.getResults()) { if (auto binOp = dyn_cast(result)) { - auto pos = dyn_cast(binOp.getLHS()).getPosition(); - if (result.getKind() == AffineExprKind::FloorDiv) { + // Check for "dim op const". + auto dimOp = dyn_cast(binOp.getLHS()); + auto conOp = dyn_cast(binOp.getRHS()); + if (!dimOp || !conOp) + return false; + // Inspect "dim / const" or "dim % const". + auto pos = dimOp.getPosition(); + if (binOp.getKind() == AffineExprKind::FloorDiv) { // Expect only one floordiv for each dimension. if (coeffientMap.find(pos) != coeffientMap.end()) return false; - coeffientMap[pos] = - dyn_cast(binOp.getRHS()).getValue(); - } else if (result.getKind() == AffineExprKind::Mod) { + // Record coefficient of the floordiv. + coeffientMap[pos] = conOp.getValue(); + } else if (binOp.getKind() == AffineExprKind::Mod) { // Expect floordiv before mod. if (coeffientMap.find(pos) == coeffientMap.end()) return false; // Expect mod to have the same coefficient as floordiv. - if (dyn_cast(binOp.getRHS()).getValue() != - coeffientMap[pos]) { + if (conOp.getValue() != coeffientMap[pos]) return false; - } } else { return false; } diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp index 6a2d7c33356f..08d37b6a9656 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.cpp @@ -147,48 +147,30 @@ static Value genSparseReducedAffineCond(OpBuilder &builder, Location loc, // Helper functions that load/store into the position buffer for slice-driven // loops. -// The sliced pointer buffer is orgnized as: -// [size, curPtr] (two metadata) + [[pLo0, pLo1, pLo2, ...], -// [pHi0, pHi1, pHi2, ...], -// [pNx0, pNx1, pNx2, ...]] +// The sliced pointer buffer is organized as: +// [[pLo0, pLo1, pLo2, ...], +// [pHi0, pHi1, pHi2, ...], +// [pNx0, pNx1, pNx2, ...]] static Value allocSlicePosBuf(OpBuilder &builder, Location loc, Value tupleCnt) { Value bufSz = MULI(tupleCnt, C_IDX(kSliceIterWidth)); // Additional two metadata {memSize, idx} at head. - bufSz = ADDI(bufSz, C_IDX(2)); return genAlloca(builder, loc, bufSz, builder.getIndexType()); } -// TODO: We should use SSA value for it. -// Gets and sets metadata. -static Value loadSlicePosPtr(OpBuilder &builder, Location loc, Value sPosBuf) { - return genIndexLoad(builder, loc, sPosBuf, C_IDX(1)); -} -static void updateSlicePosPtr(OpBuilder &builder, Location loc, Value sPosBuf, - Value pPtr) { - builder.create(loc, pPtr, sPosBuf, C_IDX(1)); -} -static Value loadSlicePosTupleNum(OpBuilder &builder, Location loc, - Value sPosBuf) { - return genIndexLoad(builder, loc, sPosBuf, C_IDX(0)); -} -static void updateSlicePosTupleNum(OpBuilder &builder, Location loc, Value num, - Value sPosBuf) { - builder.create(loc, num, sPosBuf, C_IDX(0)); -} // Gets and sets position values for slice-driven loops. enum class SlicePosKind { kLo, kHi, kNext }; static Value getSlicePosIdx(OpBuilder &builder, Location loc, Value posBuf, Value tupleIdx, SlicePosKind posKind) { Value dim = builder.create(loc, posBuf, C_IDX(0)); - Value tupleCnt = DIVUI(SUBI(dim, C_IDX(2)), C_IDX(kSliceIterWidth)); + Value tupleCnt = DIVUI(dim, C_IDX(kSliceIterWidth)); switch (posKind) { case SlicePosKind::kLo: - return ADDI(tupleIdx, C_IDX(2)); + return tupleIdx; case SlicePosKind::kHi: - return ADDI(tupleIdx, ADDI(tupleCnt, C_IDX(2))); + return ADDI(tupleIdx, tupleCnt); case SlicePosKind::kNext: - return ADDI(tupleIdx, ADDI(tupleCnt, ADDI(tupleCnt, C_IDX(2)))); + return ADDI(tupleIdx, MULI(tupleCnt, C_IDX(2))); } llvm_unreachable("unexpected kind"); } @@ -352,6 +334,9 @@ void LoopEmitter::initialize(ValueRange ts, StringAttr loopTag, bool hasOutput, this->dependentLvlMap.assign( numTensors, std::vector>>()); this->slicePosBuffer.assign(numTensors, std::vector>()); + this->sliceTupleNxStartIdx.assign(numTensors, std::vector()); + this->sliceTupleFwdCnt.assign(numTensors, std::vector()); + this->trivialSlice.assign(numTensors, std::vector()); this->sliceMeta.assign( numTensors, std::vector>>()); this->sliceStack.assign(numTensors, std::vector()); @@ -402,10 +387,13 @@ void LoopEmitter::initialize(ValueRange ts, StringAttr loopTag, bool hasOutput, dependentLvlMap[tid].assign( lvlRank, std::vector>()); slicePosBuffer[tid].assign(lvlRank, std::vector()); + sliceTupleNxStartIdx[tid].assign(lvlRank, Value()); + sliceTupleFwdCnt[tid].assign(lvlRank, Value()); + trivialSlice[tid].assign(lvlRank, false); sliceMeta[tid].assign(lvlRank, std::vector>()); sliceStack[tid].emplace_back(/*minCrd=*/Value(), /*offset=*/Value(), /*isNonEmpty*/ Value(), - std::nullopt, 0); + /*posTupleNum=*/Value(), std::nullopt, 0); if (dimGetter && !isSynTensor(tid)) { for (Level l = 0; l < lvlRank; l++) { dependentLvlMap[tid][l] = dimGetter(tid, l); @@ -814,6 +802,7 @@ std::optional LoopEmitter::genWhileLoopBody(OpBuilder &builder, assert(ivs.size() == 1); // Coord is the relative offset related to its parents. assert(sliceStack[tid].back().depth == 1 && "TODO: not yet implement"); + sliceTupleFwdCnt[tid][lvl] = SUBI(ivs[0], posits[tid][lvl]); // Update c = absOffset[lvl][depth] - absOffset[lvl][depth - 1] Value posit = ivs[0]; Value crdBuf = coordinatesBuffers[tid][lvl]; @@ -1332,6 +1321,12 @@ void LoopEmitter::enterTensorsAtDenseLvls( } else { posits[tid][lvl] = genAddress(builder, loc, tid, lvl, ADDI(info.offset, iv)); + Value fwdCnt = lvl == 0 || trivialSlice[tid][lvl] + ? C_IDX(0) + : sliceTupleFwdCnt[tid][lvl - 1]; + Value sz = sliceMeta[tid][lvl].back().first; + Value mul = MULI(fwdCnt, sz); + sliceTupleFwdCnt[tid][lvl] = ADDI(mul, iv); } levelReducedDep[tid][lvl]++; } else { @@ -1365,13 +1360,7 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc, assert(isDenseLT(lvlTypes[tid][lvl])); assert(*info.slicedOnLvl == lvl); (void)reduced; - // Resets slices pointers as the resolved slices are invalidated after we - // moves forward to the next slice. - invalidateSliceIterIdx(rewriter, loc, tid, lvl); info.minCrd = info.offset = info.isNonEmpty = Value(); - } else { - forwardsReducedSliceLevelTreeIt(rewriter, loc, tid, lvl, - constantIndex(rewriter, loc, 1)); } levelReducedDep[tid][lvl]--; } @@ -1451,63 +1440,6 @@ void LoopEmitter::exitForLoop(RewriterBase &rewriter, Location loc, } } -void LoopEmitter::forwardsReducedSliceLevelTreeIt(OpBuilder &builder, - Location loc, TensorId tid, - Level rootLvl, Value fcnt) { - auto stt = getSparseTensorType(tensors[tid]); - - // Finds a [Lvl, leafLvl) range, and all level in between are fully reduced - // level (but not resolved). Since we forward an iterator at higher level of - // the tree, the subtree need to be pruned. - Level leafLvl = rootLvl + 1; - while (leafLvl < stt.getLvlRank() && !dependentLvlMap[tid][leafLvl].empty() && - depFullyReduced(tid, leafLvl)) { - leafLvl++; - } - - Level curLvl = rootLvl + 1; - // Prunes all denses subtree. - while (curLvl < leafLvl && isDenseLT(lvlTypes[tid][curLvl])) { - // One step forward in parent level results in forwarding `slice.size` step - // in child dense level. - auto [size, stride] = sliceMeta[tid][curLvl].back(); - assert(stride == 1 && "Not yet implemented"); - fcnt = MULI(size, fcnt); - curLvl++; - } - - Value nxPosPtr = nullptr; - if (curLvl < leafLvl) { - assert(!isDenseLT(lvlTypes[tid][curLvl])); - // The first compressed level, setting up the position pointer for it. - Value sPosBuf = slicePosBuffer[tid][curLvl].back(); - // One step forwards in the parent level result in forwarding one `segment` - // in the child sparse level. - Value pPosPtr = loadSlicePosPtr(builder, loc, sPosBuf); // previous ptr - Value cPosPtr = ADDI(fcnt, pPosPtr); // current ptr - updateSlicePosPtr(builder, loc, sPosBuf, cPosPtr); - // Loads the position pointer start for next level. - nxPosPtr = - loadSlicePos(builder, loc, sPosBuf, cPosPtr, SlicePosKind::kNext); - curLvl++; - } - - // TODO: This is not always needed, but we did it unconditionally for now for - // simplicity. - // It is only needed when `curLvl` is forwarded without traversing its child - // level (e.g., the level is in a conjunctive lattices and got pruned), such - // that the position pointer is not forwarded inside the loop. - for (; curLvl < leafLvl; curLvl++) { - assert(nxPosPtr); - if (!isDenseLT(lvlTypes[tid][curLvl])) { - Value sPosBuf = slicePosBuffer[tid][curLvl].back(); - updateSlicePosPtr(builder, loc, sPosBuf, nxPosPtr); - nxPosPtr = - loadSlicePos(builder, loc, sPosBuf, nxPosPtr, SlicePosKind::kNext); - } - } -} - void LoopEmitter::exitWhileLoop(OpBuilder &builder, Location loc, MutableArrayRef reduc) { const LoopInfo &loopInfo = loopStack.back(); @@ -1557,13 +1489,6 @@ void LoopEmitter::exitWhileLoop(OpBuilder &builder, Location loc, forwarded = CMPI(eq, coords[tid][lvl], iv); operands.push_back(SELECT(forwarded, nxPos, pos)); } - { - OpBuilder::InsertionGuard guard(builder); - auto ifOp = builder.create(loc, TypeRange{}, forwarded, - /*else=*/false); - builder.setInsertionPointToStart(&ifOp.getThenRegion().front()); - forwardsReducedSliceLevelTreeIt(builder, loc, tid, lvl, one); - } // The coordinate is invalid now. coords[tid][lvl] = nullptr; @@ -1806,7 +1731,7 @@ ValueRange LoopEmitter::genUnResolvedSliceTreeTraverse( unsigned depth = frontSlice.depth - 1; Value offset = frontSlice.offset; Value sPtrBuf = slicePosBuffer[tid][firstLvl][depth]; - Value mSz = loadSlicePosTupleNum(builder, loc, sPtrBuf); + Value mSz = frontSlice.posTupleNum; outerMost = builder.create( loc, c0, mSz, c1, innerArgs, [this, tid, firstLvl, offset, sPtrBuf, &ip, &pos, @@ -1917,7 +1842,7 @@ void LoopEmitter::genResolvedSliceBegin(OpBuilder &builder, Location loc, // Dense slice begin is trivial. sliceStack[tid].emplace_back(/*minCoord=*/c0, /*offset=*/c0, /*nonEmpty=*/constantI1(builder, loc, true), - lvl, /*depth=*/1); + c0, lvl, /*depth=*/1); return; } auto [nxSz, stride] = sliceMeta[tid][lvl][1]; @@ -1933,12 +1858,12 @@ void LoopEmitter::genResolvedSliceBegin(OpBuilder &builder, Location loc, pHi = genIndexLoad(builder, loc, positionsBuffers[tid][lvl], ADDI(posits[tid][lvl - 1], c1)); } - // Fills out pIdxBuffer[tid][lvl][0] with [/*memSize =*/4, 0, pLo, pHi] - updateSlicePosTupleNum(builder, loc, c1, sPtrBuf); - updateSlicePosPtr(builder, loc, sPtrBuf, c0); + // Fills out pIdxBuffer[tid][lvl][0] with [pLo, pHi] updateSlicePos(builder, loc, sPtrBuf, pLo, c0, SlicePosKind::kLo); updateSlicePos(builder, loc, sPtrBuf, pHi, c0, SlicePosKind::kHi); - + // Slice over a resolved parent, we only need one pair of pos hi and lo to + // specify the current slice. + Value tupleNum = c1; // This is an non empty tensor if pLo < pHi. Value isNonEmpty = CMPI(ult, pLo, pHi); // The minimal coord must be at the first on ordered level. @@ -1950,7 +1875,7 @@ void LoopEmitter::genResolvedSliceBegin(OpBuilder &builder, Location loc, // FIXME: We need the relative offset related to the base slice. Value absOffset = offsetFromMinCoord(builder, loc, minCrd, nxSz, isNonEmpty); - sliceStack[tid].emplace_back(minCrd, absOffset, isNonEmpty, lvl, + sliceStack[tid].emplace_back(minCrd, absOffset, isNonEmpty, tupleNum, lvl, /*depth=*/1); } @@ -1982,8 +1907,8 @@ void LoopEmitter::genUnResolvedSliceBegin(OpBuilder &builder, Location loc, Value remSz = sliceMeta[tid][lvl][depth + 1].first; // Dense slice begin is trivial if (isDenseLT(lvlTypes[tid][lvl])) { - sliceStack[tid].emplace_back(c0, c0, constantI1(builder, loc, false), lvl, - depth + 1); + sliceStack[tid].emplace_back(c0, c0, constantI1(builder, loc, false), c0, + lvl, depth + 1); return; } @@ -2072,26 +1997,38 @@ void LoopEmitter::genUnResolvedSliceBegin(OpBuilder &builder, Location loc, Value isNonEmpty = result[0]; Value minCrd = result[1]; // Two metadata [memSize, idx]. - // TODO: Can use an SSA value for these two metadata - updateSlicePosTupleNum(builder, loc, result[2], sPtrBuf); - updateSlicePosPtr(builder, loc, sPtrBuf, c0); // FIXME: we need the relative offset related to the base slice. Value absOffset = offsetFromMinCoord(builder, loc, minCrd, remSz, isNonEmpty); - sliceStack[tid].emplace_back(minCrd, absOffset, isNonEmpty, lvl, depth + 1); + sliceStack[tid].emplace_back(minCrd, absOffset, isNonEmpty, result[2], lvl, + depth + 1); } bool LoopEmitter::genSliceBegin(OpBuilder &builder, Location loc, TensorId tid, Level lvl) { + Value curLvlIdx = C_IDX(0); if (depFullyReduced(tid, lvl)) { - // Do not need to prepare for slice driven loop on dense level after it is - // fully reduced. + if (lvl == 0 || trivialSlice[tid][lvl]) { + sliceTupleNxStartIdx[tid][lvl] = C_IDX(0); + } else { + if (isDenseLT(lvlTypes[tid][lvl])) { + sliceTupleNxStartIdx[tid][lvl] = sliceTupleNxStartIdx[tid][lvl - 1]; + } else { + assert(isCompressedLT(lvlTypes[tid][lvl])); + curLvlIdx = ADDI(sliceTupleNxStartIdx[tid][lvl - 1], + sliceTupleFwdCnt[0][lvl - 1]); + sliceTupleNxStartIdx[tid][lvl] = + loadSlicePos(builder, loc, slicePosBuffer[tid][lvl].back(), + curLvlIdx, SlicePosKind::kNext); + } + } if (isDenseLT(lvlTypes[tid][lvl])) return true; + + Value sPosBuf = slicePosBuffer[tid][lvl].back(); // If constraints on the tensor is fully resolved. We do not need to // generates slice begin any more, instead we fall back to TACO-based // algorithm to (co)iterates over the slice. - Value sPosBuf = slicePosBuffer[tid][lvl].back(); - Value tupleIdx = loadSlicePosPtr(builder, loc, sPosBuf); + Value tupleIdx = curLvlIdx; posits[tid][lvl] = loadSlicePos(builder, loc, sPosBuf, tupleIdx, SlicePosKind::kLo); highs[tid][lvl] = @@ -2150,23 +2087,16 @@ bool LoopEmitter::genSliceBegin(OpBuilder &builder, Location loc, TensorId tid, if (sliceInfo.isInitialTensor() || (lvl >= 1 && lvlFullyResolved(tid, lvl - 1))) { // First level or previous level has been full resolved. + trivialSlice[tid][lvl] = true; genResolvedSliceBegin(builder, loc, tid, lvl); } else { // The previous level has not been full resolved. + trivialSlice[tid][lvl] = false; genUnResolvedSliceBegin(builder, loc, tid, lvl); } return false; } -void LoopEmitter::invalidateSliceIterIdx(OpBuilder &builder, Location loc, - TensorId tid, Level lvl) { - for (unsigned i = 0; i <= lvl; i++) { - if (!isDenseLT(lvlTypes[tid][i]) && !dependentLvlMap[tid][i].empty()) { - updateSlicePosPtr(builder, loc, slicePosBuffer[tid][i].back(), C_IDX(0)); - } - } -} - std::tuple LoopEmitter::genSliceNextInduction(OpBuilder &builder, Location loc, TensorId tid, Level lvl) { @@ -2191,10 +2121,6 @@ LoopEmitter::genSliceNextInduction(OpBuilder &builder, Location loc, // isNonEmpty = false; // Value absOffset = info.offset; - // Resets slices pointers as the resolved slices are invalidated after we - // moves forward to the next slice. - invalidateSliceIterIdx(builder, loc, tid, lvl); - SmallVector reduc = {info.minCrd, info.isNonEmpty, absOffset}; Value sPtrBuf = slicePosBuffer[tid][lvl][info.depth - 1]; Value fastPathP = CMPI(ugt, info.minCrd, absOffset); @@ -2221,10 +2147,10 @@ LoopEmitter::genSliceNextInduction(OpBuilder &builder, Location loc, // offset = minCrd - size + 1; // } builder.setInsertionPointToStart(&ifOp.getElseRegion().front()); - reduc[2] = absOffset; // restore value. - Value mSz = loadSlicePosTupleNum(builder, loc, sPtrBuf); // memSize - reduc[0] = lvlSizes[tid][lvl]; // next min coord - reduc[1] = constantI1(builder, loc, false); // isNonEmpty + reduc[2] = absOffset; // restore value. + Value mSz = info.posTupleNum; // tuple number. + reduc[0] = lvlSizes[tid][lvl]; // next min coord + reduc[1] = constantI1(builder, loc, false); // isNonEmpty auto loopArgs = static_cast(reduc).drop_back(); auto forOp = scf::buildLoopNest( builder, loc, c0, mSz, c1, loopArgs, diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h index 0bdd9d45e6ae..fa8b0076f733 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h +++ b/mlir/lib/Dialect/SparseTensor/Transforms/LoopEmitter.h @@ -298,10 +298,10 @@ private: struct SliceInfo final { // Note that we do not need to create a actual sparse tensor slice but // instead only need to maintain the metadata of the slice. - SliceInfo(Value minCrd, Value offset, Value isNonEmpty, + SliceInfo(Value minCrd, Value offset, Value isNonEmpty, Value posTupleNum, std::optional slicedOnLvl, unsigned depth) : minCrd(minCrd), offset(offset), isNonEmpty(isNonEmpty), - slicedOnLvl(slicedOnLvl), depth(depth) { + posTupleNum(posTupleNum), slicedOnLvl(slicedOnLvl), depth(depth) { // TODO: use std::optional> assert(!slicedOnLvl || minCrd); } @@ -309,9 +309,10 @@ private: // Whether this is the tensor that has not yet been sliced. bool isInitialTensor() const { return !slicedOnLvl.has_value(); } - Value minCrd; // the minimum coordinate of the slice. - Value offset; // the *absolute* offset of the current slice. - Value isNonEmpty; // whether the slice is empty. + Value minCrd; // the minimum coordinate of the slice. + Value offset; // the *absolute* offset of the current slice. + Value isNonEmpty; // whether the slice is empty. + Value posTupleNum; // The number of position tuples used in the slice. std::optional slicedOnLvl; // the level on which the slice is done unsigned depth; // the depth (relative to dependentDimMap[tid][lvl]). }; @@ -452,11 +453,6 @@ private: return tid < lvlTypes.size() && lvl < lvlTypes[tid].size(); } - /// Forwards the (conceptual) "tree iterator" when iterating over a fully - /// reduced slice created by index-reduction. - void forwardsReducedSliceLevelTreeIt(OpBuilder &builder, Location loc, - TensorId tid, Level lvl, Value fcnt); - /// Prepares loop for iterating over `tensor[lvl]`, under the assumption /// that `tensor[0...lvl-1]` loops have already been set up. void prepareLoopOverTensorAtLvl(OpBuilder &builder, Location loc, @@ -609,11 +605,6 @@ private: void genUnResolvedSliceBegin(OpBuilder &builder, Location loc, TensorId tid, Level lvl); - /// Invalidates the index kept in slice postion buffers (by setting it to - /// zero). - /// TODO: We should instead use an SSA value for the index. - void invalidateSliceIterIdx(OpBuilder &builder, Location loc, TensorId tid, - Level lvl); /// Generates code to get the first non-empty slice of tid on lvl. /// return true if has already been resolved. bool genSliceBegin(OpBuilder &builder, Location loc, TensorId tid, Level lvl); @@ -650,17 +641,6 @@ private: std::vector> lvlTypes; // Sparse iteration information for each `(TensorId, Level)` pair. // These arrays are updated to remain current within the current loop. - // TODO: Clarify which of these are indexed by dstLvl vs srcLvl. - // - /// The collection of positions for a given element (one such collection - /// for each tensor). This is the position analogue of the "coords" - /// naming convention. - /// - /// FIXME: [CLARIFY_POSITS_LVL] It's unclear which levels are used - /// to index the `posits` array. On the one hand `genSparseCrd` - /// uses dstLvl; on the other hand `enterLoopOverTensorAtLvl`, - /// `prepareLoopOverTensorAtLvl`, and `enterCoIterationOverTensorsAtLvls` - /// uses srcLvl. So which is it? std::vector> posits; /// The collection of coordinates for a given element (one such /// collection for each tensor). @@ -693,6 +673,9 @@ private: // But they always starts with the first pidx pointing to coord > slice.offset // to avoid iteration from the beginning. std::vector>> slicePosBuffer; + std::vector> sliceTupleNxStartIdx; + std::vector> sliceTupleFwdCnt; + std::vector> trivialSlice; // The (size, stride) for each conceptual slice used for index reduction // loops. @@ -704,10 +687,6 @@ private: // sliceStack[tid] holds the generated slice stack on tid. std::vector> sliceStack; - /// TODO: not yet used, it should track the current level for each tensor - /// to help eliminate `lvls` paramters from above APIs. - /// std::vector curLvl; - // // Fields which have at most `numLoops` many entries. // diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp index 463a49f52283..cdbf4f048a00 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseBufferRewriting.cpp @@ -952,9 +952,9 @@ createQuickSort(OpBuilder &builder, ModuleOp module, func::FuncOp func, Value cond = builder.create(loc, arith::CmpIPredicate::ule, lenLow, lenHigh); + Value c0 = constantIndex(builder, loc, 0); scf::IfOp ifOp = builder.create(loc, types, cond, /*else=*/true); - Value c0 = constantIndex(builder, loc, 0); auto mayRecursion = [&](Value low, Value high, Value len) { Value cond = builder.create(loc, arith::CmpIPredicate::ne, len, c0); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp index 927fc71f84a8..5155cab772d4 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseGPUCodegen.cpp @@ -309,6 +309,10 @@ static void genGPUCode(PatternRewriter &rewriter, gpu::GPUFuncOp gpuFunc, // } Value upper = irMap.lookup(forallOp.getUpperBound()[0]); scf::ForOp forOp = rewriter.create(loc, row, upper, inc); + // The scf.for builder creates an empty block. scf.for does not allow multiple + // blocks in its region, so delete the block before `cloneRegionBefore` adds + // an additional block. + rewriter.eraseBlock(forOp.getBody()); rewriter.cloneRegionBefore(forallOp.getRegion(), forOp.getRegion(), forOp.getRegion().begin(), irMap); diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp index c94ef8b96287..488079cfe4e3 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseReinterpretMap.cpp @@ -38,16 +38,22 @@ struct DemapInsRewriter : public OpRewritePattern { LogicalResult matchAndRewrite(SourceOp op, PatternRewriter &rewriter) const override { Location loc = op.getLoc(); + // Demaps non-trivial inputs. + bool changed = false; SmallVector deMappedIns(op->getOperands()); - for (Value &in : deMappedIns) - if (auto stt = tryGetSparseTensorType(in); stt && !stt->isIdentity()) + for (Value &in : deMappedIns) { + if (auto stt = tryGetSparseTensorType(in); stt && !stt->isIdentity()) { in = rewriter.create(loc, stt->getDemappedType(), in); + changed = true; + } + } // CRTP call. OpAdaptor adaptor(deMappedIns, op); - return static_cast(this)->rewriteOp(op, adaptor, - rewriter); + LogicalResult status = + static_cast(this)->rewriteOp(op, adaptor, rewriter); + return changed ? success() : status; } }; @@ -452,11 +458,13 @@ struct GenericOpScheduler : public OpRewritePattern { } // Marks the GenericOp to avoid recursive matching. - linalgOp->setAttr(sorted, rewriter.getBoolAttr(true)); + rewriter.updateRootInPlace(linalgOp, [&]() { + linalgOp->setAttr(sorted, rewriter.getBoolAttr(true)); + }); // Already sorted. if (order.isIdentity()) - return failure(); + return success(); assert(order.isPermutation()); // `order` is orignial loop -> sorted loop map diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp index 6c9adf9fa21a..992be434fc62 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/Sparsification.cpp @@ -673,25 +673,35 @@ static void genInvariants(CodegenEnv &env, OpBuilder &builder, ExprId exp, // All exhausted at current level. if (!isCurrentLoop) return; + // Generate code for a scalarized reduction or invariant. Note that + // because custom reduction lhs may occur several times in the IR, + // we have a built-in safety for only initializing and wrapping-up + // the scalarized reduction once. OpOperand *lhs = op.getDpsInitOperand(0); if (lhs == &t) { // Start or end a scalarized reduction. if (isStart) { - Value load = env.isCustomReduc() ? env.getCustomRedId() - : genTensorLoad(env, builder, exp); - env.startReduc(exp, load); + if (env.isCustomReduc()) { + if (!env.isReduc()) + env.startReduc(exp, env.getCustomRedId()); + } else { + env.startReduc(exp, genTensorLoad(env, builder, exp)); + } if (env.hasSparseOutput()) env.setValidLexInsert(constantI1(builder, env.op().getLoc(), false)); } else { - genTensorStore(env, builder, exp, env.endReduc()); - env.clearValidLexInsert(); + if (!env.isCustomReduc() || env.isReduc()) + genTensorStore(env, builder, exp, env.endReduc()); + if (env.hasSparseOutput()) + env.clearValidLexInsert(); } } else { // Start or end loop invariant hoisting of a tensor load. - if (isStart) + if (isStart) { env.merger().setExprValue(exp, genTensorLoad(env, builder, exp)); - else + } else { env.merger().clearExprValue(exp); + } } } else if (env.exp(exp).kind != TensorExp::Kind::kInvariant && env.exp(exp).kind != TensorExp::Kind::kLoopVar && diff --git a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp index 55f813df78b8..ec4c41c0000a 100644 --- a/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp +++ b/mlir/lib/Dialect/Tensor/IR/TensorOps.cpp @@ -168,6 +168,41 @@ static llvm::SmallBitVector getDroppedDims(ArrayRef reducedShape, return droppedDims; } +/// Given a ranked tensor type and a range of values that defines its dynamic +/// dimension sizes, turn all dynamic sizes that have a constant value into +/// static dimension sizes. +static RankedTensorType +foldDynamicToStaticDimSizes(RankedTensorType type, ValueRange dynamicSizes, + SmallVector &foldedDynamicSizes) { + SmallVector staticShape(type.getShape().begin(), + type.getShape().end()); + assert(type.getNumDynamicDims() == + static_cast(dynamicSizes.size()) && + "incorrect number of dynamic sizes"); + + // Compute new static and dynamic sizes. + unsigned ctr = 0; + for (int64_t i = 0, e = type.getRank(); i < e; ++i) { + if (type.isDynamicDim(i)) { + Value dynamicSize = dynamicSizes[ctr++]; + std::optional cst = getConstantIntValue(dynamicSize); + if (cst.has_value()) { + // Dynamic size must be non-negative. + if (cst.value() < 0) { + foldedDynamicSizes.push_back(dynamicSize); + continue; + } + staticShape[i] = *cst; + } else { + foldedDynamicSizes.push_back(dynamicSize); + } + } + } + + return RankedTensorType::get(staticShape, type.getElementType(), + type.getEncoding()); +} + //===----------------------------------------------------------------------===// // BitcastOp //===----------------------------------------------------------------------===// @@ -889,37 +924,16 @@ struct ReplaceEmptyTensorStaticShapeDims : OpRewritePattern { LogicalResult matchAndRewrite(EmptyOp op, PatternRewriter &rewriter) const override { - SmallVector staticShape(op.getType().getShape().begin(), - op.getType().getShape().end()); - SmallVector dynamicSizes; - - // Compute new static and dynamic sizes. - unsigned ctr = 0; - bool changedType = false; - for (int64_t i = 0; i < op.getType().getRank(); ++i) { - if (op.getType().isDynamicDim(i)) { - Value dynamicSize = op.getDynamicSizes()[ctr++]; - std::optional cst = getConstantIntValue(dynamicSize); - if (cst.has_value()) { - // dynamic size must be non-negative. - if (cst.value() < 0) - return failure(); - staticShape[i] = *cst; - changedType = true; - } else { - dynamicSizes.push_back(dynamicSize); - } - } - } + SmallVector foldedDynamicSizes; + RankedTensorType foldedTensorType = foldDynamicToStaticDimSizes( + op.getType(), op.getDynamicSizes(), foldedDynamicSizes); // Stop here if no dynamic size was promoted to static. - if (!changedType) + if (foldedTensorType == op.getType()) return failure(); - auto tensorType = RankedTensorType::get( - staticShape, op.getType().getElementType(), op.getType().getEncoding()); - auto newOp = - rewriter.create(op.getLoc(), tensorType, dynamicSizes); + auto newOp = rewriter.create(op.getLoc(), foldedTensorType, + foldedDynamicSizes); rewriter.replaceOpWithNewOp(op, op.getType(), newOp); return success(); } @@ -1347,28 +1361,6 @@ LogicalResult GenerateOp::reifyResultShapes( return success(); } -/// Extract operands and shape from a tensor with dynamic extents. -static void operandsAndShape(TensorType resultType, - Operation::operand_range dynamicExtents, - SmallVectorImpl &newOperands, - SmallVectorImpl &newShape) { - auto operandsIt = dynamicExtents.begin(); - for (int64_t dim : resultType.getShape()) { - if (!ShapedType::isDynamic(dim)) { - newShape.push_back(dim); - continue; - } - APInt index; - if (!matchPattern(*operandsIt, m_ConstantInt(&index))) { - newShape.push_back(ShapedType::kDynamic); - newOperands.push_back(*operandsIt++); - continue; - } - newShape.push_back(index.getSExtValue()); - operandsIt++; - } -} - LogicalResult GenerateOp::verify() { // Ensure that the tensor type has as many dynamic dimensions as are // specified by the operands. @@ -1376,14 +1368,6 @@ LogicalResult GenerateOp::verify() { if (getNumOperands() != resultType.getNumDynamicDims()) return emitError("must have as many index operands as dynamic extents " "in the result type"); - // Ensure operands are non-negative. - SmallVector newOperands; - SmallVector newShape; - operandsAndShape(resultType, getDynamicExtents(), newOperands, newShape); - for (int64_t newdim : newShape) { - if (newdim < 0 && !ShapedType::isDynamic(newdim)) - return emitError("tensor dimensions must be non-negative"); - } return success(); } @@ -1433,34 +1417,24 @@ namespace { struct StaticTensorGenerate : public OpRewritePattern { using OpRewritePattern::OpRewritePattern; - LogicalResult matchAndRewrite(GenerateOp tensorFromElements, + LogicalResult matchAndRewrite(GenerateOp generateOp, PatternRewriter &rewriter) const final { - auto resultType = - llvm::cast(tensorFromElements.getResult().getType()); + SmallVector foldedDynamicSizes; + RankedTensorType foldedTensorType = foldDynamicToStaticDimSizes( + generateOp.getType(), generateOp.getDynamicExtents(), + foldedDynamicSizes); - if (resultType.hasStaticShape()) - return failure(); - - Operation::operand_range dynamicExtents = - tensorFromElements.getDynamicExtents(); - SmallVector newOperands; - SmallVector newShape; - operandsAndShape(resultType, dynamicExtents, newOperands, newShape); - - if (!hasValidSizesOffsets(newShape)) - return failure(); - - if (newOperands.size() == tensorFromElements.getDynamicExtents().size()) + // Stop here if no dynamic size was promoted to static. + if (foldedTensorType == generateOp.getType()) return failure(); - auto loc = tensorFromElements.getLoc(); - auto newOp = rewriter.create( - loc, RankedTensorType::get(newShape, resultType.getElementType()), - newOperands); - rewriter.inlineRegionBefore(tensorFromElements.getBody(), newOp.getBody(), + auto loc = generateOp.getLoc(); + auto newOp = + rewriter.create(loc, foldedTensorType, foldedDynamicSizes); + rewriter.inlineRegionBefore(generateOp.getBody(), newOp.getBody(), newOp.getBody().begin()); - rewriter.replaceOpWithNewOp(tensorFromElements, resultType, - newOp); + rewriter.replaceOpWithNewOp(generateOp, + generateOp.getType(), newOp); return success(); } }; diff --git a/mlir/lib/Dialect/Utils/IndexingUtils.cpp b/mlir/lib/Dialect/Utils/IndexingUtils.cpp index f4e29539214b..bb8a0d5912d7 100644 --- a/mlir/lib/Dialect/Utils/IndexingUtils.cpp +++ b/mlir/lib/Dialect/Utils/IndexingUtils.cpp @@ -70,7 +70,7 @@ SmallVector delinearizeImpl(ExprType linearIndex, //===----------------------------------------------------------------------===// SmallVector mlir::computeSuffixProduct(ArrayRef sizes) { - assert(llvm::all_of(sizes, [](int64_t s) { return s > 0; }) && + assert(llvm::all_of(sizes, [](int64_t s) { return s >= 0; }) && "sizes must be nonnegative"); int64_t unit = 1; return ::computeSuffixProductImpl(sizes, unit); diff --git a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp index c462b23e1133..540959b486db 100644 --- a/mlir/lib/Dialect/Vector/IR/VectorOps.cpp +++ b/mlir/lib/Dialect/Vector/IR/VectorOps.cpp @@ -3815,6 +3815,11 @@ ParseResult TransferReadOp::parse(OpAsmParser &parser, OperationState &result) { if (llvm::dyn_cast(shapedType.getElementType())) return parser.emitError( maskInfo.location, "does not support masks with vector element type"); + if (vectorType.getRank() != permMap.getNumResults()) { + return parser.emitError(typesLoc, + "expected the same rank for the vector and the " + "results of the permutation map"); + } // Instead of adding the mask type as an op type, compute it based on the // vector type and the permutation map (to keep the type signature small). auto maskType = inferTransferOpMaskType(vectorType, permMap); @@ -4181,6 +4186,11 @@ ParseResult TransferWriteOp::parse(OpAsmParser &parser, if (llvm::dyn_cast(shapedType.getElementType())) return parser.emitError( maskInfo.location, "does not support masks with vector element type"); + if (vectorType.getRank() != permMap.getNumResults()) { + return parser.emitError(typesLoc, + "expected the same rank for the vector and the " + "results of the permutation map"); + } auto maskType = inferTransferOpMaskType(vectorType, permMap); if (parser.resolveOperand(maskInfo, maskType, result.operands)) return failure(); diff --git a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp index 4d43a76c4a4e..97f6caca1b25 100644 --- a/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/LowerVectorTranspose.cpp @@ -334,24 +334,22 @@ public: return rewriter.notifyMatchFailure( op, "Options specifies lowering to shuffle"); - if (vectorTransformOptions.useShapeCast) { - // Replace: - // vector.transpose %0, [1, 0] : vector> to - // vector<1xnxelty> - // with: - // vector.shape_cast %0 : vector> to vector<1xnxelty> - // - // Source with leading unit dim (inverse) is also replaced. Unit dim must - // be fixed. Non-unit can be scalable. - if (resType.getRank() == 2 && - ((resType.getShape().front() == 1 && - !resType.getScalableDims().front()) || - (resType.getShape().back() == 1 && - !resType.getScalableDims().back())) && - transp == ArrayRef({1, 0})) { - rewriter.replaceOpWithNewOp(op, resType, input); - return success(); - } + // Replace: + // vector.transpose %0, [1, 0] : vector> to + // vector<1xnxelty> + // with: + // vector.shape_cast %0 : vector> to vector<1xnxelty> + // + // Source with leading unit dim (inverse) is also replaced. Unit dim must + // be fixed. Non-unit can be scalable. + if (resType.getRank() == 2 && + ((resType.getShape().front() == 1 && + !resType.getScalableDims().front()) || + (resType.getShape().back() == 1 && + !resType.getScalableDims().back())) && + transp == ArrayRef({1, 0})) { + rewriter.replaceOpWithNewOp(op, resType, input); + return success(); } if (inputType.isScalable()) diff --git a/mlir/lib/IR/AsmPrinter.cpp b/mlir/lib/IR/AsmPrinter.cpp index 4b76dcf7f8a9..1f7cbf349255 100644 --- a/mlir/lib/IR/AsmPrinter.cpp +++ b/mlir/lib/IR/AsmPrinter.cpp @@ -16,7 +16,9 @@ #include "mlir/IR/AsmState.h" #include "mlir/IR/Attributes.h" #include "mlir/IR/Builders.h" +#include "mlir/IR/BuiltinAttributes.h" #include "mlir/IR/BuiltinDialect.h" +#include "mlir/IR/BuiltinTypeInterfaces.h" #include "mlir/IR/BuiltinTypes.h" #include "mlir/IR/Dialect.h" #include "mlir/IR/DialectImplementation.h" @@ -27,6 +29,7 @@ #include "mlir/IR/Operation.h" #include "mlir/IR/Verifier.h" #include "llvm/ADT/APFloat.h" +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/DenseMap.h" #include "llvm/ADT/MapVector.h" #include "llvm/ADT/STLExtras.h" @@ -44,6 +47,7 @@ #include "llvm/Support/SaveAndRestore.h" #include "llvm/Support/Threading.h" #include "llvm/Support/raw_ostream.h" +#include #include #include @@ -425,6 +429,8 @@ public: void popCyclicPrinting(); + void printDimensionList(ArrayRef shape); + protected: void printOptionalAttrDict(ArrayRef attrs, ArrayRef elidedAttrs = {}, @@ -1860,6 +1866,20 @@ private: // Allow direct access to the impl fields. friend AsmState; }; + +template +void printDimensionList(raw_ostream &stream, Range &&shape) { + llvm::interleave( + shape, stream, + [&stream](const auto &dimSize) { + if (ShapedType::isDynamic(dimSize)) + stream << "?"; + else + stream << dimSize; + }, + "x"); +} + } // namespace detail } // namespace mlir @@ -2576,13 +2596,9 @@ void AsmPrinter::Impl::printTypeImpl(Type type) { }) .Case([&](RankedTensorType tensorTy) { os << "tensor<"; - for (int64_t dim : tensorTy.getShape()) { - if (ShapedType::isDynamic(dim)) - os << '?'; - else - os << dim; + printDimensionList(tensorTy.getShape()); + if (!tensorTy.getShape().empty()) os << 'x'; - } printType(tensorTy.getElementType()); // Only print the encoding attribute value if set. if (tensorTy.getEncoding()) { @@ -2598,13 +2614,9 @@ void AsmPrinter::Impl::printTypeImpl(Type type) { }) .Case([&](MemRefType memrefTy) { os << "memref<"; - for (int64_t dim : memrefTy.getShape()) { - if (ShapedType::isDynamic(dim)) - os << '?'; - else - os << dim; + printDimensionList(memrefTy.getShape()); + if (!memrefTy.getShape().empty()) os << 'x'; - } printType(memrefTy.getElementType()); MemRefLayoutAttrInterface layout = memrefTy.getLayout(); if (!llvm::isa(layout) || !layout.isIdentity()) { @@ -2735,6 +2747,10 @@ LogicalResult AsmPrinter::Impl::pushCyclicPrinting(const void *opaquePointer) { void AsmPrinter::Impl::popCyclicPrinting() { state.popCyclicPrinting(); } +void AsmPrinter::Impl::printDimensionList(ArrayRef shape) { + detail::printDimensionList(os, shape); +} + //===--------------------------------------------------------------------===// // AsmPrinter //===--------------------------------------------------------------------===// @@ -2800,6 +2816,10 @@ void AsmPrinter::printResourceHandle(const AsmDialectResourceHandle &resource) { impl->printResourceHandle(resource); } +void AsmPrinter::printDimensionList(ArrayRef shape) { + detail::printDimensionList(getStream(), shape); +} + LogicalResult AsmPrinter::pushCyclicPrinting(const void *opaquePointer) { return impl->pushCyclicPrinting(opaquePointer); } @@ -3911,3 +3931,47 @@ void Block::printAsOperand(raw_ostream &os, AsmState &state) { OperationPrinter printer(os, state.getImpl()); printer.printBlockName(this); } + +//===--------------------------------------------------------------------===// +// Custom printers +//===--------------------------------------------------------------------===// +namespace mlir { + +void printDimensionList(OpAsmPrinter &printer, Operation *op, + ArrayRef dimensions) { + if (dimensions.empty()) + printer << "["; + printer.printDimensionList(dimensions); + if (dimensions.empty()) + printer << "]"; +} + +ParseResult parseDimensionList(OpAsmParser &parser, + DenseI64ArrayAttr &dimensions) { + // Empty list case denoted by "[]". + if (succeeded(parser.parseOptionalLSquare())) { + if (failed(parser.parseRSquare())) { + return parser.emitError(parser.getCurrentLocation()) + << "Failed parsing dimension list."; + } + dimensions = + DenseI64ArrayAttr::get(parser.getContext(), ArrayRef()); + return success(); + } + + // Non-empty list case. + SmallVector shapeArr; + if (failed(parser.parseDimensionList(shapeArr, true, false))) { + return parser.emitError(parser.getCurrentLocation()) + << "Failed parsing dimension list."; + } + if (shapeArr.empty()) { + return parser.emitError(parser.getCurrentLocation()) + << "Failed parsing dimension list. Did you mean an empty list? It " + "must be denoted by \"[]\"."; + } + dimensions = DenseI64ArrayAttr::get(parser.getContext(), shapeArr); + return success(); +} + +} // namespace mlir diff --git a/mlir/lib/Pass/Pass.cpp b/mlir/lib/Pass/Pass.cpp index 658f8844b428..810d6a357d52 100644 --- a/mlir/lib/Pass/Pass.cpp +++ b/mlir/lib/Pass/Pass.cpp @@ -36,11 +36,21 @@ using namespace mlir::detail; // PassExecutionAction //===----------------------------------------------------------------------===// +PassExecutionAction::PassExecutionAction(ArrayRef irUnits, + const Pass &pass) + : Base(irUnits), pass(pass) {} + void PassExecutionAction::print(raw_ostream &os) const { os << llvm::formatv("`{0}` running `{1}` on Operation `{2}`", tag, pass.getName(), getOp()->getName()); } +Operation *PassExecutionAction::getOp() const { + ArrayRef irUnits = getContextIRUnits(); + return irUnits.empty() ? nullptr + : llvm::dyn_cast_if_present(irUnits[0]); +} + //===----------------------------------------------------------------------===// // Pass //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Pass/PassDetail.h b/mlir/lib/Pass/PassDetail.h index 727607146a68..0e964b6d6d36 100644 --- a/mlir/lib/Pass/PassDetail.h +++ b/mlir/lib/Pass/PassDetail.h @@ -15,26 +15,6 @@ #include "llvm/Support/FormatVariadic.h" namespace mlir { -/// Encapsulate the "action" of executing a single pass, used for the MLIR -/// tracing infrastructure. -struct PassExecutionAction : public tracing::ActionImpl { - using Base = tracing::ActionImpl; - PassExecutionAction(ArrayRef irUnits, const Pass &pass) - : Base(irUnits), pass(pass) {} - static constexpr StringLiteral tag = "pass-execution"; - void print(raw_ostream &os) const override; - const Pass &getPass() const { return pass; } - Operation *getOp() const { - ArrayRef irUnits = getContextIRUnits(); - return irUnits.empty() ? nullptr - : llvm::dyn_cast_if_present(irUnits[0]); - } - -public: - const Pass &pass; - Operation *op; -}; - namespace detail { //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Target/LLVMIR/ModuleImport.cpp b/mlir/lib/Target/LLVMIR/ModuleImport.cpp index 4bdffa572e31..7c51ee7420f9 100644 --- a/mlir/lib/Target/LLVMIR/ModuleImport.cpp +++ b/mlir/lib/Target/LLVMIR/ModuleImport.cpp @@ -619,6 +619,19 @@ void ModuleImport::setNonDebugMetadataAttrs(llvm::Instruction *inst, } } +void ModuleImport::setIntegerOverflowFlagsAttr(llvm::Instruction *inst, + Operation *op) const { + auto iface = cast(op); + + IntegerOverflowFlags value = {}; + value = bitEnumSet(value, IntegerOverflowFlags::nsw, inst->hasNoSignedWrap()); + value = + bitEnumSet(value, IntegerOverflowFlags::nuw, inst->hasNoUnsignedWrap()); + + auto attr = IntegerOverflowFlagsAttr::get(op->getContext(), value); + iface->setAttr(iface.getIntegerOverflowAttrName(), attr); +} + void ModuleImport::setFastmathFlagsAttr(llvm::Instruction *inst, Operation *op) const { auto iface = cast(op); diff --git a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp index 8e2bfe557c55..7decbce018a8 100644 --- a/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp +++ b/mlir/lib/Transforms/Utils/GreedyPatternRewriteDriver.cpp @@ -581,9 +581,6 @@ void GreedyPatternRewriteDriver::notifyOperationReplaced( }); if (config.listener) config.listener->notifyOperationReplaced(op, replacement); - for (auto result : op->getResults()) - for (auto *user : result.getUsers()) - addToWorklist(user); } LogicalResult GreedyPatternRewriteDriver::notifyMatchFailure( diff --git a/mlir/python/CMakeLists.txt b/mlir/python/CMakeLists.txt index 55731943fb78..585918afc263 100644 --- a/mlir/python/CMakeLists.txt +++ b/mlir/python/CMakeLists.txt @@ -335,7 +335,8 @@ declare_mlir_dialect_python_bindings( TD_FILE dialects/OpenMPOps.td SOURCES dialects/openmp.py - DIALECT_NAME omp) + DIALECT_NAME omp + DEPENDS omp_common_td) declare_mlir_dialect_python_bindings( ADD_TO_PARENT MLIRPythonSources.Dialects diff --git a/mlir/python/mlir/dialects/_ods_common.py b/mlir/python/mlir/dialects/_ods_common.py index 60ce83c09f17..1685124fbccd 100644 --- a/mlir/python/mlir/dialects/_ods_common.py +++ b/mlir/python/mlir/dialects/_ods_common.py @@ -134,3 +134,7 @@ def get_op_result_or_op_results( # see the typing.Type doc string. _U = _TypeVar("_U", bound=_cext.ir.Value) SubClassValueT = _Type[_U] + +ResultValueTypeTuple = _cext.ir.Operation, _cext.ir.OpView, _cext.ir.Value +ResultValueT = _Union[ResultValueTypeTuple] +VariadicResultValueT = _Union[ResultValueT, _Sequence[ResultValueT]] diff --git a/mlir/python/mlir/dialects/affine.py b/mlir/python/mlir/dialects/affine.py index 26e827009bc0..913cea61105c 100644 --- a/mlir/python/mlir/dialects/affine.py +++ b/mlir/python/mlir/dialects/affine.py @@ -3,8 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception from ._affine_ops_gen import * -from ._affine_ops_gen import _Dialect, AffineForOp -from .arith import constant +from ._affine_ops_gen import _Dialect try: from ..ir import * @@ -12,6 +11,9 @@ try: get_op_result_or_value as _get_op_result_or_value, get_op_results_or_values as _get_op_results_or_values, _cext as _ods_cext, + ResultValueTypeTuple as _ResultValueTypeTuple, + ResultValueT as _ResultValueT, + VariadicResultValueT as _VariadicResultValueT, ) except ImportError as e: raise RuntimeError("Error loading imports from extension module") from e @@ -21,17 +23,17 @@ from typing import Optional, Sequence, Union @_ods_cext.register_operation(_Dialect, replace=True) class AffineForOp(AffineForOp): - """Specialization for the Affine for op class""" + """Specialization for the Affine for op class.""" def __init__( self, - lower_bound, - upper_bound, - step, - iter_args: Optional[Union[Operation, OpView, Sequence[Value]]] = None, + lower_bound: Union[int, _ResultValueT, AffineMap], + upper_bound: Optional[Union[int, _ResultValueT, AffineMap]], + step: Optional[Union[int, Attribute]] = None, + iter_args: Optional[_ResultValueT] = None, *, - lower_bound_operands=[], - upper_bound_operands=[], + lower_bound_operands: Optional[_VariadicResultValueT] = None, + upper_bound_operands: Optional[_VariadicResultValueT] = None, loc=None, ip=None, ): @@ -43,25 +45,57 @@ class AffineForOp(AffineForOp): - `iter_args` is a list of additional loop-carried arguments or an operation producing them as results. - `lower_bound_operands` is the list of arguments to substitute the dimensions, - then symbols in the `lower_bound` affine map, in an increasing order + then symbols in the `lower_bound` affine map, in an increasing order. - `upper_bound_operands` is the list of arguments to substitute the dimensions, - then symbols in the `upper_bound` affine map, in an increasing order + then symbols in the `upper_bound` affine map, in an increasing order. """ + if lower_bound_operands is None: + lower_bound_operands = [] + if upper_bound_operands is None: + upper_bound_operands = [] + + if step is None: + step = 1 + + bounds_operands = [lower_bound_operands, upper_bound_operands] + bounds = [lower_bound, upper_bound] + bounds_names = ["lower", "upper"] + for i, name in enumerate(bounds_names): + if isinstance(bounds[i], int): + bounds[i] = AffineMap.get_constant(bounds[i]) + elif isinstance(bounds[i], _ResultValueTypeTuple): + if len(bounds_operands[i]): + raise ValueError( + f"Either a concrete {name} bound or an AffineMap in combination " + f"with {name} bound operands, but not both, is supported." + ) + if ( + isinstance(bounds[i], (OpView, Operation)) + and len(bounds[i].results) > 1 + ): + raise ValueError( + f"Only a single concrete value is supported for {name} bound." + ) + + bounds_operands[i].append(_get_op_result_or_value(bounds[i])) + bounds[i] = AffineMap.get_identity(1) + + if not isinstance(bounds[i], AffineMap): + raise ValueError( + f"{name} bound must be int | ResultValueT | AffineMap." + ) + if len(bounds_operands[i]) != bounds[i].n_inputs: + raise ValueError( + f"Wrong number of {name} bound operands passed to AffineForOp; " + + f"Expected {bounds[i].n_inputs}, got {len(bounds_operands[i])}." + ) + + lower_bound, upper_bound = bounds + if iter_args is None: iter_args = [] iter_args = _get_op_results_or_values(iter_args) - if len(lower_bound_operands) != lower_bound.n_inputs: - raise ValueError( - f"Wrong number of lower bound operands passed to AffineForOp. " - + "Expected {lower_bound.n_symbols}, got {len(lower_bound_operands)}." - ) - - if len(upper_bound_operands) != upper_bound.n_inputs: - raise ValueError( - f"Wrong number of upper bound operands passed to AffineForOp. " - + "Expected {upper_bound.n_symbols}, got {len(upper_bound_operands)}." - ) results = [arg.type for arg in iter_args] super().__init__( @@ -71,7 +105,7 @@ class AffineForOp(AffineForOp): inits=list(iter_args), lowerBoundMap=AffineMapAttr.get(lower_bound), upperBoundMap=AffineMapAttr.get(upper_bound), - step=IntegerAttr.get(IndexType.get(), step), + step=step, loc=loc, ip=ip, ) @@ -98,37 +132,18 @@ class AffineForOp(AffineForOp): def for_( start, - stop=None, + stop, step=None, iter_args: Optional[Sequence[Value]] = None, *, loc=None, ip=None, ): - if step is None: - step = 1 - if stop is None: - stop = start - start = 0 - params = [start, stop] - for i, p in enumerate(params): - if isinstance(p, int): - p = constant(IntegerAttr.get(IndexType.get(), p)) - elif isinstance(p, float): - raise ValueError(f"{p=} must be int.") - params[i] = p - - start, stop = params - s0 = AffineSymbolExpr.get(0) - lbmap = AffineMap.get(0, 1, [s0]) - ubmap = AffineMap.get(0, 1, [s0]) for_op = AffineForOp( - lbmap, - ubmap, + start, + stop, step, iter_args=iter_args, - lower_bound_operands=[start], - upper_bound_operands=[stop], loc=loc, ip=ip, ) diff --git a/mlir/test/CAPI/ir.c b/mlir/test/CAPI/ir.c index 315458a08b61..a9850c0a132e 100644 --- a/mlir/test/CAPI/ir.c +++ b/mlir/test/CAPI/ir.c @@ -1970,6 +1970,15 @@ int testOperands(void) { fprintf(stderr, "\n"); // CHECK: Second replacement use owner: "dummy.op2" + MlirOpOperand use5 = mlirValueGetFirstUse(constTwoValue); + MlirOpOperand use6 = mlirOpOperandGetNextUse(use5); + if (!mlirValueEqual(mlirOpOperandGetValue(use5), + mlirOpOperandGetValue(use6))) { + fprintf(stderr, + "ERROR: First and second operand should share the same value\n"); + return 5; + } + mlirOperationDestroy(op); mlirOperationDestroy(op2); mlirOperationDestroy(constZero); diff --git a/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir b/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir index 4837a0cce634..f0c4512cbfdc 100644 --- a/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir +++ b/mlir/test/Conversion/MathToLibm/convert-to-libm.mlir @@ -1,5 +1,7 @@ // RUN: mlir-opt %s -convert-math-to-libm -canonicalize | FileCheck %s +// CHECK-DAG: @acos(f64) -> f64 attributes {llvm.readnone} +// CHECK-DAG: @acosf(f32) -> f32 attributes {llvm.readnone} // CHECK-DAG: @atan(f64) -> f64 attributes {llvm.readnone} // CHECK-DAG: @atanf(f32) -> f32 attributes {llvm.readnone} // CHECK-DAG: @erf(f64) -> f64 attributes {llvm.readnone} @@ -29,6 +31,43 @@ // CHECK-DAG: @ceil(f64) -> f64 attributes {llvm.readnone} // CHECK-DAG: @ceilf(f32) -> f32 attributes {llvm.readnone} +// CHECK-LABEL: func @acos_caller +// CHECK-SAME: %[[FLOAT:.*]]: f32 +// CHECK-SAME: %[[DOUBLE:.*]]: f64 +func.func @acos_caller(%float: f32, %double: f64) -> (f32, f64) { + // CHECK-DAG: %[[FLOAT_RESULT:.*]] = call @acosf(%[[FLOAT]]) : (f32) -> f32 + %float_result = math.acos %float : f32 + // CHECK-DAG: %[[DOUBLE_RESULT:.*]] = call @acos(%[[DOUBLE]]) : (f64) -> f64 + %double_result = math.acos %double : f64 + // CHECK: return %[[FLOAT_RESULT]], %[[DOUBLE_RESULT]] + return %float_result, %double_result : f32, f64 +} + +// CHECK-LABEL: func @acos_vec_caller( +// CHECK-SAME: %[[VAL_0:.*]]: vector<2xf32>, +// CHECK-SAME: %[[VAL_1:.*]]: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) { +// CHECK-DAG: %[[CVF:.*]] = arith.constant dense<0.000000e+00> : vector<2xf32> +// CHECK-DAG: %[[CVD:.*]] = arith.constant dense<0.000000e+00> : vector<2xf64> +// CHECK: %[[IN0_F32:.*]] = vector.extract %[[VAL_0]][0] : f32 from vector<2xf32> +// CHECK: %[[OUT0_F32:.*]] = call @acosf(%[[IN0_F32]]) : (f32) -> f32 +// CHECK: %[[VAL_8:.*]] = vector.insert %[[OUT0_F32]], %[[CVF]] [0] : f32 into vector<2xf32> +// CHECK: %[[IN1_F32:.*]] = vector.extract %[[VAL_0]][1] : f32 from vector<2xf32> +// CHECK: %[[OUT1_F32:.*]] = call @acosf(%[[IN1_F32]]) : (f32) -> f32 +// CHECK: %[[VAL_11:.*]] = vector.insert %[[OUT1_F32]], %[[VAL_8]] [1] : f32 into vector<2xf32> +// CHECK: %[[IN0_F64:.*]] = vector.extract %[[VAL_1]][0] : f64 from vector<2xf64> +// CHECK: %[[OUT0_F64:.*]] = call @acos(%[[IN0_F64]]) : (f64) -> f64 +// CHECK: %[[VAL_14:.*]] = vector.insert %[[OUT0_F64]], %[[CVD]] [0] : f64 into vector<2xf64> +// CHECK: %[[IN1_F64:.*]] = vector.extract %[[VAL_1]][1] : f64 from vector<2xf64> +// CHECK: %[[OUT1_F64:.*]] = call @acos(%[[IN1_F64]]) : (f64) -> f64 +// CHECK: %[[VAL_17:.*]] = vector.insert %[[OUT1_F64]], %[[VAL_14]] [1] : f64 into vector<2xf64> +// CHECK: return %[[VAL_11]], %[[VAL_17]] : vector<2xf32>, vector<2xf64> +// CHECK: } +func.func @acos_vec_caller(%float: vector<2xf32>, %double: vector<2xf64>) -> (vector<2xf32>, vector<2xf64>) { + %float_result = math.acos %float : vector<2xf32> + %double_result = math.acos %double : vector<2xf64> + return %float_result, %double_result : vector<2xf32>, vector<2xf64> +} + // CHECK-LABEL: func @atan_caller // CHECK-SAME: %[[FLOAT:.*]]: f32 // CHECK-SAME: %[[DOUBLE:.*]]: f64 diff --git a/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir b/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir index fa9fff2dad66..962ed7de584a 100644 --- a/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir +++ b/mlir/test/Conversion/VectorToGPU/vector-to-mma-ops.mlir @@ -460,3 +460,33 @@ func.func @cast_f16_to_f32_write(%arg0: memref<16x16xf16>, %arg1: memref<16x16xf vector.transfer_write %cast, %arg3[%c0, %c0] {in_bounds = [true, true]} : vector<16x16xf32>, memref<16x16xf32> return } + +// ----- + +#map1 = affine_map<(d0, d1, d2) -> (d0, d2)> +#map2 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map3 = affine_map<(d0, d1, d2) -> (d0, d1)> + +// CHECK-DAG: #[[$MAP:.+]] = affine_map<(d0, d1) -> (d1, d0)> +// CHECK-LABEL: func @fold_transpose_into_transfer_read( +// CHECK-SAME: %[[ALLOC:.+]]: memref<64x128xf16> +// CHECK-DAG: %[[C0:.+]] = arith.constant 0 : index +// CHECK-DAG: %[[CST:.+]] = arith.constant 0.000000e+00 : f16 +// CHECK: %[[READ:.+]] = vector.transfer_read %[[ALLOC]][%[[C0]], %[[C0]]], %[[CST]] {in_bounds = [true, true], permutation_map = #[[$MAP]]} +// CHECK: %[[EXTF1:.+]] = arith.extf %[[READ]] +// CHECK-NOT: vector.transpose +// CHECK: %[[RESULT:.+]] = vector.contract +func.func @fold_transpose_into_transfer_read(%alloc: memref<64x128xf16>, %vector: vector<32x128xf16>, %alloc2: memref<32x64xf32>) { + %c0 = arith.constant 0 : index + %cst = arith.constant 0.000000e+00 : f16 + %init = arith.constant dense<0.000000e+00> : vector<32x64xf32> + %0 = vector.transfer_read %alloc[%c0, %c0], %cst {in_bounds = [true, true]} : memref<64x128xf16>, vector<64x128xf16> + %1 = arith.extf %0 : vector<64x128xf16> to vector<64x128xf32> + %2 = arith.extf %vector : vector<32x128xf16> to vector<32x128xf32> + %3 = vector.transpose %1, [1, 0] : vector<64x128xf32> to vector<128x64xf32> + %4 = vector.contract {indexing_maps = [#map1, #map2, #map3], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind} %2, %3, %init : vector<32x128xf32>, vector<128x64xf32> into vector<32x64xf32> + vector.transfer_write %4, %alloc2[%c0, %c0] {in_bounds = [true, true]} : vector<32x64xf32>, memref<32x64xf32> + return +} + +// ----- diff --git a/mlir/test/Dialect/Arith/int-range-interface.mlir b/mlir/test/Dialect/Arith/int-range-interface.mlir index 4c5f4095225d..02a9827d19d8 100644 --- a/mlir/test/Dialect/Arith/int-range-interface.mlir +++ b/mlir/test/Dialect/Arith/int-range-interface.mlir @@ -730,3 +730,29 @@ func.func @extui_uses_unsigned(%arg0 : i32) -> i1 { %4 = arith.andi %2, %3 : i1 func.return %4 : i1 } + +/// Catch a bug that caused a crash in getLoopBoundFromFold when +/// SparseConstantPropagation is loaded in the solver. + +// CHECK-LABEL: func.func @caller( +// CHECK-SAME: %[[VAL_0:.*]]: memref) { +// CHECK: call @callee(%[[VAL_0]]) : (memref) -> () +// CHECK: return +// CHECK: } +func.func @caller(%arg0: memref) { + call @callee(%arg0) : (memref) -> () + return +} + +// CHECK-LABEL: func.func private @callee( +// CHECK-SAME: %[[VAL_0:.*]]: memref) { +// CHECK: return +// CHECK: } +func.func private @callee(%arg0: memref) { + %c1 = arith.constant 1 : index + %c0 = arith.constant 0 : index + %0 = affine.load %arg0[0] : memref + scf.for %arg1 = %c0 to %0 step %c1 { + } + return +} diff --git a/mlir/test/Dialect/Complex/ops.mlir b/mlir/test/Dialect/Complex/ops.mlir index 1050ad0dcd53..96f17b2898c8 100644 --- a/mlir/test/Dialect/Complex/ops.mlir +++ b/mlir/test/Dialect/Complex/ops.mlir @@ -11,6 +11,9 @@ func.func @ops(%f: f32) { // CHECK: complex.constant [1.{{.*}} : f32, -1.{{.*}} : f32] : complex %cst_f32 = complex.constant [0.1 : f32, -1.0 : f32] : complex + // CHECK: complex.constant [true, false] : complex + %cst_i1 = complex.constant [1 : i1, 0 : i1] : complex + // CHECK: %[[C:.*]] = complex.create %[[F]], %[[F]] : complex %complex = complex.create %f, %f : complex diff --git a/mlir/test/Dialect/LLVMIR/roundtrip.mlir b/mlir/test/Dialect/LLVMIR/roundtrip.mlir index 42589972ef2b..594c3de91815 100644 --- a/mlir/test/Dialect/LLVMIR/roundtrip.mlir +++ b/mlir/test/Dialect/LLVMIR/roundtrip.mlir @@ -34,6 +34,16 @@ func.func @ops(%arg0: i32, %arg1: f32, %vptrcmp = llvm.icmp "ne" %arg5, %arg5 : !llvm.vec<2 x ptr> %typecheck_vptrcmp = llvm.add %vptrcmp, %vptrcmp : vector<2 x i1> +// Integer overflow flags +// CHECK: {{.*}} = llvm.add %[[I32]], %[[I32]] overflow : i32 +// CHECK: {{.*}} = llvm.sub %[[I32]], %[[I32]] overflow : i32 +// CHECK: {{.*}} = llvm.mul %[[I32]], %[[I32]] overflow : i32 +// CHECK: {{.*}} = llvm.shl %[[I32]], %[[I32]] overflow : i32 + %add_flag = llvm.add %arg0, %arg0 overflow : i32 + %sub_flag = llvm.sub %arg0, %arg0 overflow : i32 + %mul_flag = llvm.mul %arg0, %arg0 overflow : i32 + %shl_flag = llvm.shl %arg0, %arg0 overflow : i32 + // Floating point binary operations. // // CHECK: {{.*}} = llvm.fadd %[[FLOAT]], %[[FLOAT]] : f32 diff --git a/mlir/test/Dialect/LLVMIR/type-consistency.mlir b/mlir/test/Dialect/LLVMIR/type-consistency.mlir index 1504a98e6f8c..3a1ab924ebda 100644 --- a/mlir/test/Dialect/LLVMIR/type-consistency.mlir +++ b/mlir/test/Dialect/LLVMIR/type-consistency.mlir @@ -151,6 +151,20 @@ llvm.func @index_to_struct(%arg: i32) { // ----- +// CHECK-LABEL: llvm.func @no_crash_on_negative_gep_index +llvm.func @no_crash_on_negative_gep_index() { + %0 = llvm.mlir.constant(1.000000e+00 : f16) : f16 + %1 = llvm.mlir.constant(1 : i32) : i32 + // CHECK: %[[ALLOCA:.*]] = llvm.alloca %{{.*}} x !llvm.struct<"foo", (i32, i32, i32)> + %2 = llvm.alloca %1 x !llvm.struct<"foo", (i32, i32, i32)> : (i32) -> !llvm.ptr + // CHECK: llvm.getelementptr %[[ALLOCA]][-1] : (!llvm.ptr) -> !llvm.ptr, f32 + %3 = llvm.getelementptr %2[-1] : (!llvm.ptr) -> !llvm.ptr, f32 + llvm.store %0, %3 : f16, !llvm.ptr + llvm.return +} + +// ----- + // CHECK-LABEL: llvm.func @coalesced_store_ints // CHECK-SAME: %[[ARG:.*]]: i64 llvm.func @coalesced_store_ints(%arg: i64) { diff --git a/mlir/test/Dialect/Linalg/transform-op-match.mlir b/mlir/test/Dialect/Linalg/transform-op-match.mlir index 7d48b1f403b3..fed3c007d9b6 100644 --- a/mlir/test/Dialect/Linalg/transform-op-match.mlir +++ b/mlir/test/Dialect/Linalg/transform-op-match.mlir @@ -43,6 +43,44 @@ module attributes {transform.with_named_sequence} { // ----- +func.func @by_operand_type() { + %c2 = arith.constant 2.0: f32 + %v = arith.constant 8: i32 + %r1 = math.fpowi %c2, %v : f32, i32 + // expected-remark @below {{matched op name}} + %r2 = arith.addf %c2, %c2 : f32 + // expected-remark @below {{matched op name}} + %r3 = arith.fptoui %r2 : f32 to i32 + return +} + +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %match_name1 = transform.structured.match + ops{["arith.fptoui"]} filter_operand_types = [f32] in %arg1 : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %match_name1, "matched op name" : !transform.any_op + transform.test_consume_operand %match_name1 : !transform.any_op + + %match_name2 = transform.structured.match + ops{["arith.addf"]} filter_operand_types = [f32] in %arg1 : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %match_name2, "matched op name" : !transform.any_op + transform.test_consume_operand %match_name2 : !transform.any_op + + %no_match_name1 = transform.structured.match + ops{["arith.fptoui"]} filter_operand_types = [i32] in %arg1 : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %no_match_name1, "should not match" : !transform.any_op + transform.test_consume_operand %no_match_name1 : !transform.any_op + + %no_match_name2 = transform.structured.match + ops{["math.fpowi"]} filter_operand_types = [f32] in %arg1 : (!transform.any_op) -> !transform.any_op + transform.test_print_remark_at_operand %no_match_name2, "should not match" : !transform.any_op + transform.test_consume_operand %no_match_name2 : !transform.any_op + transform.yield + } +} + +// ----- + func.func @foo(%a: tensor<4x4xf32>, %b: tensor<4x4xf32>, %c: tensor<4x4xf32>) { %c0 = arith.constant 0.0 : f32 // expected-remark @below {{tileable}} diff --git a/mlir/test/Dialect/Linalg/transpose-conv2d.mlir b/mlir/test/Dialect/Linalg/transpose-conv2d.mlir index 4655a261d986..409c36778ab7 100644 --- a/mlir/test/Dialect/Linalg/transpose-conv2d.mlir +++ b/mlir/test/Dialect/Linalg/transpose-conv2d.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -transform-interpreter -verify-diagnostics | FileCheck %s +// RUN: mlir-opt %s -transform-interpreter | FileCheck %s // CHECK-LABEL: @conv_2d_nhwc_fhwc_f64 // CHECK-SAME: (%[[INPUT:.+]]: tensor<1x4x4x6xf64>, %[[FILTER:.+]]: tensor<8x2x2x6xf64>, %[[INIT:.+]]: tensor<1x2x2x8xf64>) -> tensor<1x2x2x8xf64> { diff --git a/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir b/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir index ab0c78a8ba76..28b700430059 100644 --- a/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir +++ b/mlir/test/Dialect/MemRef/expand-strided-metadata.mlir @@ -1494,3 +1494,23 @@ func.func @extract_strided_metadata_of_cast_unranked( index, index, index, index } + + +// ----- +memref.global "private" @dynamicShmem : memref<0xf16,3> + +// CHECK-LABEL: func @zero_sized_memred +func.func @zero_sized_memred(%arg0: f32) -> (memref, index,index,index) { + %c0 = arith.constant 0 : index + %dynamicMem = memref.get_global @dynamicShmem : memref<0xf16, 3> + + // CHECK: %[[BASE:.*]] = memref.get_global @dynamicShmem : memref<0xf16, 3> + // CHECK: %[[CAST:.*]] = memref.reinterpret_cast %[[BASE]] to offset: [0], sizes: [], strides: [] : memref<0xf16, 3> to memref + // CHECK: return %[[CAST]] + + %base_buffer, %offset, %sizes, %strides = memref.extract_strided_metadata %dynamicMem : memref<0xf16, 3> -> memref, index, index, index + return %base_buffer, %offset, + %sizes, %strides : + memref, index, + index, index +} \ No newline at end of file diff --git a/mlir/test/Dialect/Mesh/canonicalization.mlir b/mlir/test/Dialect/Mesh/canonicalization.mlir index 5802d198d368..baee9faa645c 100644 --- a/mlir/test/Dialect/Mesh/canonicalization.mlir +++ b/mlir/test/Dialect/Mesh/canonicalization.mlir @@ -1,6 +1,6 @@ // RUN: mlir-opt --canonicalize %s | FileCheck %s -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) // CHECK-LABEL: func @all_reduce_empty_mesh_axes func.func @all_reduce_empty_mesh_axes( diff --git a/mlir/test/Dialect/Mesh/invalid.mlir b/mlir/test/Dialect/Mesh/invalid.mlir index 2999668f770b..a26e3950186e 100644 --- a/mlir/test/Dialect/Mesh/invalid.mlir +++ b/mlir/test/Dialect/Mesh/invalid.mlir @@ -6,16 +6,16 @@ mesh.cluster @mesh0(rank = 0) // ----- // expected-error@+1 {{rank of dim_sizes is not expected to be larger than rank of cluster}} -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 3, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x3x4) // ----- -// expected-error@+1 {{dimension size of a mesh cluster is expected to be non-negative}} -mesh.cluster @mesh0(rank = 2, dim_sizes = [-1]) +// expected-error@+1 {{custom op 'mesh.cluster' Failed parsing dimension list. Did you mean an empty list? It must be denoted by "[]".}} +mesh.cluster @mesh0(rank = 2, dim_sizes = -1) // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @mesh_axis_duplicated_different_subarray( // expected-error@+1 {{mesh axis duplicated}} @@ -26,7 +26,7 @@ func.func @mesh_axis_duplicated_different_subarray( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @mesh_axis_duplicated_same_subarray( // expected-error@+1 {{mesh axis duplicated}} @@ -37,7 +37,7 @@ func.func @mesh_axis_duplicated_same_subarray( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @mesh_axis_duplicated_bewteen_split_and_partial( // expected-error@+1 {{mesh axis duplicated}} @@ -48,7 +48,7 @@ func.func @mesh_axis_duplicated_bewteen_split_and_partial( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @mesh_axis_negtive_in_split_part( // expected-error@+1 {{mesh axis is expected to be non-negative}} @@ -59,7 +59,7 @@ func.func @mesh_axis_negtive_in_split_part( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @mesh_axis_negtive_in_partial( // expected-error@+1 {{mesh axis is expected to be non-negative}} @@ -80,7 +80,7 @@ func.func @all_reduce_invalid_mesh_symbol( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @all_reduce_invalid_mesh_axis( %arg0 : tensor<4xf32>) -> tensor<4xf64> { @@ -92,7 +92,7 @@ func.func @all_reduce_invalid_mesh_axis( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @all_reduce_duplicate_mesh_axis( %arg0 : tensor<4xf32>) -> tensor<4xf64> { @@ -104,7 +104,7 @@ func.func @all_reduce_duplicate_mesh_axis( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @all_reduce_invalid_tensor_dimension_size( %arg0 : tensor<4xf32>) -> tensor<5xf64> { @@ -125,7 +125,7 @@ func.func @all_gather_invalid_mesh_symbol( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @all_gather_invalid_mesh_axis( %arg0 : tensor<4xf32>) -> tensor<4xf32> { @@ -137,7 +137,7 @@ func.func @all_gather_invalid_mesh_axis( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 2x4) func.func @all_reduce_duplicate_mesh_axis( %arg0 : tensor<4xf32>) -> tensor<4xf32> { @@ -149,7 +149,7 @@ func.func @all_reduce_duplicate_mesh_axis( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [1]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 1) func.func @all_gather_invalid_non_gather_axis_dimension_size( %arg0 : tensor<3x4xf32>) -> tensor<3x5xf32> { @@ -161,7 +161,7 @@ func.func @all_gather_invalid_non_gather_axis_dimension_size( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [1, 2]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 1x2) func.func @all_gather_invalid_gather_axis_dimension_size( %arg0 : tensor<3x4xf32>) -> tensor<3x5xf32> { @@ -173,7 +173,7 @@ func.func @all_gather_invalid_gather_axis_dimension_size( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [1]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 1) func.func @all_gather_invalid_gather_axis_dynamic_dimension( %arg0 : tensor) -> tensor<3xf32> { @@ -185,7 +185,7 @@ func.func @all_gather_invalid_gather_axis_dynamic_dimension( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [1]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 1) func.func @all_gather_invalid_gather_axis( %arg0 : tensor<3xf32>) -> tensor<3xf32> { @@ -197,7 +197,7 @@ func.func @all_gather_invalid_gather_axis( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [1]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 1) func.func @all_gather_invalid_negative_gather_axis( %arg0 : tensor<3xf32>) -> tensor<3xf32> { @@ -220,7 +220,7 @@ func.func @all_to_all_invalid_mesh_symbol( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [1]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 1) func.func @all_to_all_duplicate_mesh_axis( %arg0 : tensor<3x6xi8>) -> tensor<3x6xi8> { @@ -233,7 +233,7 @@ func.func @all_to_all_duplicate_mesh_axis( // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [0, 1]) +mesh.cluster @mesh0(rank = 2, dim_sizes = ?x1) func.func @all_to_all_invalid_non_dynamic_result_dimension_induced_by_dynamic_device_group( %arg0 : tensor<3x6xi8>) -> tensor<3x6xi8> { @@ -246,7 +246,7 @@ func.func @all_to_all_invalid_non_dynamic_result_dimension_induced_by_dynamic_de // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [1, 1]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 1x1) func.func @all_to_all_invalid_non_dynamic_result_split_dimension_induced_by_dynamic_operand_dimension( %arg0 : tensor) -> tensor<3x?xi8> { @@ -259,7 +259,7 @@ func.func @all_to_all_invalid_non_dynamic_result_split_dimension_induced_by_dyna // ----- -mesh.cluster @mesh0(rank = 2, dim_sizes = [1, 1]) +mesh.cluster @mesh0(rank = 2, dim_sizes = 1x1) func.func @all_to_all_invalid_non_dynamic_result_concat_dimension_induced_by_dynamic_operand_dimension( %arg0 : tensor<3x?xi8>) -> tensor { @@ -272,7 +272,7 @@ func.func @all_to_all_invalid_non_dynamic_result_concat_dimension_induced_by_dyn // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 3) func.func @all_to_all_invalid_non_dynamic_result_concat_dimension_size( %arg0 : tensor<3x2xi8>) -> tensor<1x7xi8> { @@ -285,7 +285,7 @@ func.func @all_to_all_invalid_non_dynamic_result_concat_dimension_size( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 3) func.func @all_to_all_invalid_non_dynamic_result_split_dimension_size( %arg0 : tensor<3x2xi8>) -> tensor<2x6xi8> { @@ -298,7 +298,7 @@ func.func @all_to_all_invalid_non_dynamic_result_split_dimension_size( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 3) func.func @reduce_scatter_duplicate_mesh_axis( %arg0 : tensor) -> tensor { @@ -310,7 +310,7 @@ func.func @reduce_scatter_duplicate_mesh_axis( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 3) func.func @reduce_scatter_invalid_dynamic_dimension( %arg0 : tensor) -> tensor<2xf64> { @@ -322,7 +322,7 @@ func.func @reduce_scatter_invalid_dynamic_dimension( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 3) func.func @reduce_scatter_invalid_static_dimension_size( %arg0 : tensor<3xf32>) -> tensor<2xf64> { @@ -334,7 +334,7 @@ func.func @reduce_scatter_invalid_static_dimension_size( // ----- -mesh.cluster @mesh0(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh0(rank = 1, dim_sizes = 3) func.func @reduce_scatter_invalid_operand_static_dimension_size( %arg0 : tensor<4xf32>) -> tensor { diff --git a/mlir/test/Dialect/Mesh/ops.mlir b/mlir/test/Dialect/Mesh/ops.mlir index 5b264bc88dfc..78ce276a7b33 100644 --- a/mlir/test/Dialect/Mesh/ops.mlir +++ b/mlir/test/Dialect/Mesh/ops.mlir @@ -1,18 +1,21 @@ // RUN: mlir-opt %s | mlir-opt | FileCheck %s // CHECK: mesh.cluster @mesh0 -mesh.cluster @mesh0(rank = 3, dim_sizes = [2, 2, 4]) +mesh.cluster @mesh0(rank = 3, dim_sizes = 2x2x4) -// CHECK: mesh.cluster @mesh1 -mesh.cluster @mesh1(rank = 2, dim_sizes = [4]) +// CHECK: mesh.cluster @mesh1(rank = 2, dim_sizes = 4) +mesh.cluster @mesh1(rank = 2, dim_sizes = 4) -// CHECK: mesh.cluster @mesh2 -mesh.cluster @mesh2(rank = 2, dim_sizes = [0, 4]) +// CHECK: mesh.cluster @mesh2(rank = 2, dim_sizes = ?x4) +mesh.cluster @mesh2(rank = 2, dim_sizes = ?x4) // CHECK: mesh.cluster @mesh3 mesh.cluster @mesh3(rank = 2) -mesh.cluster @mesh4(rank = 1, dim_sizes = [3]) +mesh.cluster @mesh4(rank = 1, dim_sizes = 3) + +// CHECK: mesh.cluster @mesh5(rank = 1) +mesh.cluster @mesh5(rank = 1, dim_sizes = []) // CHECK-LABEL: func @mesh_shard_encoding_fully_replicated func.func @mesh_shard_encoding_fully_replicated( diff --git a/mlir/test/Dialect/Mesh/sharding-propagation.mlir b/mlir/test/Dialect/Mesh/sharding-propagation.mlir index bda407b52bfd..30bbd5c6619e 100644 --- a/mlir/test/Dialect/Mesh/sharding-propagation.mlir +++ b/mlir/test/Dialect/Mesh/sharding-propagation.mlir @@ -1,7 +1,7 @@ // RUN: mlir-opt -sharding-propagation %s | FileCheck %s mesh.cluster @mesh_1d(rank = 1) -mesh.cluster @mesh_2d(rank = 2, dim_sizes = [2, 4]) +mesh.cluster @mesh_2d(rank = 2, dim_sizes = 2x4) mesh.cluster @mesh_3d(rank = 3) // CHECK-LABEL: func.func @element_wise_empty_sharding_info diff --git a/mlir/test/Dialect/NVGPU/transform-pipeline-shared.mlir b/mlir/test/Dialect/NVGPU/transform-pipeline-shared.mlir index 42b072374261..e959949babd9 100644 --- a/mlir/test/Dialect/NVGPU/transform-pipeline-shared.mlir +++ b/mlir/test/Dialect/NVGPU/transform-pipeline-shared.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s --transform-interpreter --split-input-file --verify-diagnostics | FileCheck %s +// RUN: mlir-opt %s --transform-interpreter -canonicalize --split-input-file --verify-diagnostics | FileCheck %s func.func @simple_depth_2_unpeeled(%global: memref, %result: memref ) { %c0 = arith.constant 0 : index @@ -78,15 +78,19 @@ module attributes {transform.with_named_sequence} { // CHECK-LABEL: @async_depth_2_predicated // CHECK-SAME: %[[GLOBAL:.+]]: memref -func.func @async_depth_2_predicated(%global: memref) { +func.func @async_depth_2_predicated(%global: memref, %alloc_size: index) { %c0 = arith.constant 0 : index %c98 = arith.constant 98 : index %c100 = arith.constant 100 : index - %c200 = arith.constant 200 : index - // CHECK: %[[C4:.+]] = arith.constant 4 + // CHECK-DAG: %[[C4:.+]] = arith.constant 4 + // CHECK-DAG: %[[C90:.+]] = arith.constant 90 + // CHECK-DAG: %[[C96:.+]] = arith.constant 96 + // CHECK-DAG: %[[C8:.+]] = arith.constant 8 + // CHECK-DAG: %[[C2:.+]] = arith.constant 2 + // CHECK-DAG: %[[C0:.+]] = arith.constant 0 %c4 = arith.constant 4 : index // CHECK: %[[SHARED:.+]] = memref.alloc{{.*}} #gpu.address_space - %shared = memref.alloc(%c200) : memref> + %shared = memref.alloc(%alloc_size) : memref> %c0f = arith.constant 0.0 : f32 // CHECK: %[[TOKEN0:.+]] = nvgpu.device_async_copy // CHECK: %[[TOKEN1:.+]] = nvgpu.device_async_copy @@ -95,16 +99,11 @@ func.func @async_depth_2_predicated(%global: memref) { // CHECK-SAME: %[[ITER_ARG1:.+]] = %[[TOKEN1]] scf.for %i = %c0 to %c98 step %c4 { // Condition for the predication "select" below. - // CHECK: %[[C90:.+]] = arith.constant 90 // CHECK: %[[CMP0:.+]] = arith.cmpi slt, %[[I]], %[[C90]] // CHECK: nvgpu.device_async_wait %[[ITER_ARG0]] {numGroups = 1 - // Original "select" with updated induction variable. - // CHECK: %[[C96:.+]] = arith.constant 96 - // CHECK: %[[C8:.+]] = arith.constant 8 // CHECK: %[[I_PLUS_8:.+]] = arith.addi %[[I]], %[[C8]] // CHECK: %[[CMP1:.+]] = arith.cmpi slt, %[[I_PLUS_8]], %[[C96]] - // CHECK: %[[C2:.+]] = arith.constant 2 // CHECK: %[[SELECTED0:.+]] = arith.select %[[CMP1]], %[[C4]], %[[C2]] %c96 = arith.constant 96 : index %cond = arith.cmpi slt, %i, %c96 : index @@ -113,14 +112,11 @@ func.func @async_depth_2_predicated(%global: memref) { // Updated induction variables (two more) for the device_async_copy below. // These are generated repeatedly by the pipeliner. - // CHECK: %[[C8_2:.+]] = arith.constant 8 - // CHECK: %[[I_PLUS_8_2:.+]] = arith.addi %[[I]], %[[C8_2]] - // CHECK: %[[C8_3:.+]] = arith.constant 8 - // CHECK: %[[I_PLUS_8_3:.+]] = arith.addi %[[I]], %[[C8_3]] + // CHECK: %[[I_PLUS_8_2:.+]] = arith.addi %[[I]], %[[C8]] + // CHECK: %[[I_PLUS_8_3:.+]] = arith.addi %[[I]], %[[C8]] // The second "select" is generated by predication and selects 0 for // the two last iterations. - // CHECK: %[[C0:.+]] = arith.constant 0 // CHECK: %[[SELECTED1:.+]] = arith.select %[[CMP0]], %[[SELECTED0]], %[[C0]] // CHECK: %[[ASYNC_TOKEN:.+]] = nvgpu.device_async_copy %[[GLOBAL]][%[[I_PLUS_8_3]]], %[[SHARED]][%[[I_PLUS_8_2]]], 4, %[[SELECTED1]] %token = nvgpu.device_async_copy %global[%i], %shared[%i], 4, %read_size diff --git a/mlir/test/Dialect/OpenACC/ops.mlir b/mlir/test/Dialect/OpenACC/ops.mlir index c06e5a596923..05b0450c7fb9 100644 --- a/mlir/test/Dialect/OpenACC/ops.mlir +++ b/mlir/test/Dialect/OpenACC/ops.mlir @@ -974,7 +974,7 @@ func.func @testupdateop(%a: memref, %b: memref, %c: memref) -> () acc.update async(%idxValue: index) dataOperands(%0: memref) acc.update wait_devnum(%i64Value: i64) wait(%i32Value, %idxValue : i32, index) dataOperands(%0: memref) acc.update if(%ifCond) dataOperands(%0: memref) - acc.update dataOperands(%0: memref) attributes {acc.device_types = [#acc.device_type]} + acc.update dataOperands(%0: memref) attributes {acc.device_types = [#acc.device_type]} acc.update dataOperands(%0, %1, %2 : memref, memref, memref) acc.update dataOperands(%0, %1, %2 : memref, memref, memref) attributes {async} acc.update dataOperands(%0, %1, %2 : memref, memref, memref) attributes {wait} @@ -993,7 +993,7 @@ func.func @testupdateop(%a: memref, %b: memref, %c: memref) -> () // CHECK: acc.update async([[IDXVALUE]] : index) dataOperands(%{{.*}} : memref) // CHECK: acc.update wait_devnum([[I64VALUE]] : i64) wait([[I32VALUE]], [[IDXVALUE]] : i32, index) dataOperands(%{{.*}} : memref) // CHECK: acc.update if([[IFCOND]]) dataOperands(%{{.*}} : memref) -// CHECK: acc.update dataOperands(%{{.*}} : memref) attributes {acc.device_types = [#acc.device_type]} +// CHECK: acc.update dataOperands(%{{.*}} : memref) attributes {acc.device_types = [#acc.device_type]} // CHECK: acc.update dataOperands(%{{.*}}, %{{.*}}, %{{.*}} : memref, memref, memref) // CHECK: acc.update dataOperands(%{{.*}}, %{{.*}}, %{{.*}} : memref, memref, memref) attributes {async} // CHECK: acc.update dataOperands(%{{.*}}, %{{.*}}, %{{.*}} : memref, memref, memref) attributes {wait} diff --git a/mlir/test/Dialect/SCF/canonicalize.mlir b/mlir/test/Dialect/SCF/canonicalize.mlir index 9dbf8d5dab11..41e028028616 100644 --- a/mlir/test/Dialect/SCF/canonicalize.mlir +++ b/mlir/test/Dialect/SCF/canonicalize.mlir @@ -773,56 +773,6 @@ func.func @remove_empty_parallel_loop(%lb: index, %ub: index, %s: index) { // ----- -func.func private @process(%0 : memref<128x128xf32>) -func.func private @process_tensor(%0 : tensor<128x128xf32>) -> memref<128x128xf32> - -// CHECK-LABEL: last_value -// CHECK-SAME: %[[T0:[0-9a-z]*]]: tensor<128x128xf32> -// CHECK-SAME: %[[T1:[0-9a-z]*]]: tensor<128x128xf32> -// CHECK-SAME: %[[T2:[0-9a-z]*]]: tensor<128x128xf32> -// CHECK-SAME: %[[M0:[0-9a-z]*]]: memref<128x128xf32> -func.func @last_value(%t0: tensor<128x128xf32>, %t1: tensor<128x128xf32>, - %t2: tensor<128x128xf32>, %m0: memref<128x128xf32>, - %lb : index, %ub : index, %step : index) - -> (tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32>) -{ - // CHECK-NEXT: %[[M1:.*]] = bufferization.to_memref %[[T1]] : memref<128x128xf32> - // CHECK-NEXT: %[[FOR_RES:.*]] = scf.for {{.*}} iter_args(%[[BBARG_T2:.*]] = %[[T2]]) -> (tensor<128x128xf32>) { - %0:3 = scf.for %arg0 = %lb to %ub step %step iter_args(%arg1 = %t0, %arg2 = %t1, %arg3 = %t2) - -> (tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32>) - { - %m1 = bufferization.to_memref %arg2 : memref<128x128xf32> - - // CHECK-NEXT: call @process(%[[M0]]) : (memref<128x128xf32>) -> () - func.call @process(%m0) : (memref<128x128xf32>) -> () - - // CHECK-NEXT: call @process(%[[M1]]) : (memref<128x128xf32>) -> () - func.call @process(%m1) : (memref<128x128xf32>) -> () - - // This does not hoist (fails the bbArg has at most a single check). - // CHECK-NEXT: %[[T:.*]] = func.call @process_tensor(%[[BBARG_T2]]) : (tensor<128x128xf32>) -> memref<128x128xf32> - // CHECK-NEXT: %[[YIELD_T:.*]] = bufferization.to_tensor %[[T:.*]] - %m2 = func.call @process_tensor(%arg3): (tensor<128x128xf32>) -> memref<128x128xf32> - %3 = bufferization.to_tensor %m2 : memref<128x128xf32> - - // All this stuff goes away, incrementally - %1 = bufferization.to_tensor %m0 : memref<128x128xf32> - %2 = bufferization.to_tensor %m1 : memref<128x128xf32> - - // CHECK-NEXT: scf.yield %[[YIELD_T]] : tensor<128x128xf32> - scf.yield %1, %2, %3 : tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32> - - // CHECK-NEXT: } - } - - // CHECK-NEXT: %[[R0:.*]] = bufferization.to_tensor %[[M0]] : memref<128x128xf32> - // CHECK-NEXT: %[[R1:.*]] = bufferization.to_tensor %[[M1]] : memref<128x128xf32> - // CHECK-NEXT: return %[[R0]], %[[R1]], %[[FOR_RES]] : tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32> - return %0#0, %0#1, %0#2 : tensor<128x128xf32>, tensor<128x128xf32>, tensor<128x128xf32> -} - -// ----- - // CHECK-LABEL: fold_away_iter_with_no_use_and_yielded_input // CHECK-SAME: %[[A0:[0-9a-z]*]]: i32 func.func @fold_away_iter_with_no_use_and_yielded_input(%arg0 : i32, diff --git a/mlir/test/Dialect/SCF/loop-pipelining.mlir b/mlir/test/Dialect/SCF/loop-pipelining.mlir index 4cd686d2cdb8..8a57ddccfee6 100644 --- a/mlir/test/Dialect/SCF/loop-pipelining.mlir +++ b/mlir/test/Dialect/SCF/loop-pipelining.mlir @@ -723,3 +723,50 @@ func.func @stage_0_value_escape(%A: memref, %result: memref) { memref.store %r, %result[%c1] : memref return } + +// ----- + +// NOEPILOGUE-LABEL: dynamic_loop( +// NOEPILOGUE-SAME: %[[A:.*]]: memref, %[[R:.*]]: memref, %[[LB:.+]]: index, %[[UB:.+]]: index, %[[STEP:.+]]: index) { +// NOEPILOGUE-DAG: %[[C2:.+]] = arith.constant 2 : index +// NOEPILOGUE-DAG: %[[CSTF:.+]] = arith.constant 1.000000e+00 : f32 +// Prologue: +// NOEPILOGUE: %[[P_I0:.+]] = arith.cmpi slt, %[[LB]], %[[UB]] : index +// NOEPILOGUE: %[[L0:.+]] = scf.if %[[P_I0]] -> (f32) { +// NOEPILOGUE-NEXT: memref.load %[[A]][%[[LB]]] : memref +// NOEPILOGUE: %[[IV1:.+]] = arith.addi %[[LB]], %[[STEP]] : index +// NOEPILOGUE: %[[P_I1:.+]] = arith.cmpi slt, %[[IV1]], %[[UB]] : index +// NOEPILOGUE: %[[IV1_2:.+]] = arith.addi %[[LB]], %[[STEP]] : index +// NOEPILOGUE: %[[V0:.+]] = scf.if %[[P_I0]] -> (f32) { +// NOEPILOGUE-NEXT: arith.addf %[[L0]], %[[CSTF]] : f32 +// NOEPILOGUE: %[[L1:.+]] = scf.if %[[P_I1]] -> (f32) { +// NOEPILOGUE-NEXT: memref.load %[[A]][%[[IV1_2]]] : memref +// NOEPILOGUE: scf.for %[[IV2:.+]] = %[[LB]] to %[[UB]] step %[[STEP]] iter_args(%[[V1:.+]] = %[[V0]], %[[L2:.+]] = %[[L1]]) -> (f32, f32) { +// NOEPILOGUE-DAG: %[[S2:.+]] = arith.muli %[[STEP]], %[[C2]] : index +// NOEPILOGUE-DAG: %[[IT2:.+]] = arith.subi %[[UB]], %[[S2]] : index +// NOEPILOGUE-DAG: %[[P_I2:.+]] = arith.cmpi slt, %[[IV2]], %[[IT2]] : index +// NOEPILOGUE-DAG: %[[IT3:.+]] = arith.subi %[[UB]], %[[STEP]] : index +// NOEPILOGUE-DAG: %[[P_I3:.+]] = arith.cmpi slt, %[[IV2]], %[[IT3]] : index +// NOEPILOGUE: memref.store %[[V1]], %[[R]][%[[IV2]]] : memref +// NOEPILOGUE: %[[V2:.+]] = scf.if %[[P_I3]] -> (f32) { +// NOEPILOGUE: arith.addf %[[L2]], %[[CSTF]] : f32 +// NOEPILOGUE: %[[IT4:.+]] = arith.muli %[[STEP]], %[[C2]] : index +// NOEPILOGUE: %[[IV3:.+]] = arith.addi %[[IV2]], %[[IT4]] : index +// NOEPILOGUE: %[[L3:.+]] = scf.if %[[P_I2]] -> (f32) { +// NOEPILOGUE: memref.load %[[A]][%[[IV3]]] : memref +// NOEPILOGUE: scf.yield %[[V2]], %[[L3]] : f32, f32 + +// In case dynamic loop pipelining is off check that the transformation didn't +// apply. +// CHECK-LABEL: dynamic_loop( +// CHECK-NOT: memref.load +// CHECK: scf.for +func.func @dynamic_loop(%A: memref, %result: memref, %lb: index, %ub: index, %step: index) { + %cf = arith.constant 1.0 : f32 + scf.for %i0 = %lb to %ub step %step { + %A_elem = memref.load %A[%i0] { __test_pipelining_stage__ = 0, __test_pipelining_op_order__ = 2 } : memref + %A1_elem = arith.addf %A_elem, %cf { __test_pipelining_stage__ = 1, __test_pipelining_op_order__ = 1 } : f32 + memref.store %A1_elem, %result[%i0] { __test_pipelining_stage__ = 2, __test_pipelining_op_order__ = 0 } : memref + } { __test_pipelining_loop__ } + return +} diff --git a/mlir/test/Dialect/SPIRV/IR/bit-ops.mlir b/mlir/test/Dialect/SPIRV/IR/bit-ops.mlir index 82a2316f6c78..f3f0ebf60f46 100644 --- a/mlir/test/Dialect/SPIRV/IR/bit-ops.mlir +++ b/mlir/test/Dialect/SPIRV/IR/bit-ops.mlir @@ -149,14 +149,16 @@ func.func @bitwise_or_float(%arg0: f16, %arg1: f16) -> f16 { //===----------------------------------------------------------------------===// func.func @bitwise_xor_scalar(%arg: i32) -> i32 { + %c1 = spirv.Constant 1 : i32 // using constant to avoid folding // CHECK: spirv.BitwiseXor - %0 = spirv.BitwiseXor %arg, %arg : i32 + %0 = spirv.BitwiseXor %c1, %arg : i32 return %0 : i32 } func.func @bitwise_xor_vector(%arg: vector<4xi32>) -> vector<4xi32> { + %c1 = spirv.Constant dense<1> : vector<4xi32> // using constant to avoid folding // CHECK: spirv.BitwiseXor - %0 = spirv.BitwiseXor %arg, %arg : vector<4xi32> + %0 = spirv.BitwiseXor %c1, %arg : vector<4xi32> return %0 : vector<4xi32> } diff --git a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir index 867ddf3c8017..29bea91ce461 100644 --- a/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir +++ b/mlir/test/Dialect/SPIRV/Transforms/canonicalize.mlir @@ -1139,6 +1139,372 @@ func.func @convert_logical_or_true_false_vector(%arg: vector<3xi1>) -> (vector<3 // ----- +//===----------------------------------------------------------------------===// +// spirv.LeftShiftLogical +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @lsl_x_0 +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @lsl_x_0(%arg0 : i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %c0 = spirv.Constant 0 : i32 + %cv0 = spirv.Constant dense<0> : vector<3xi32> + + %0 = spirv.ShiftLeftLogical %arg0, %c0 : i32, i32 + %1 = spirv.ShiftLeftLogical %arg1, %cv0 : vector<3xi32>, vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @lsl_shift_overflow +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @lsl_shift_overflow(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + // CHECK-DAG: %[[C32:.*]] = spirv.Constant 32 + // CHECK-DAG: %[[CV:.*]] = spirv.Constant dense<[6, 18, 128]> + %c32 = spirv.Constant 32 : i32 + %cv = spirv.Constant dense<[6, 18, 128]> : vector<3xi32> + + // CHECK: %0 = spirv.ShiftLeftLogical %[[ARG0]], %[[C32]] + // CHECK: %1 = spirv.ShiftLeftLogical %[[ARG1]], %[[CV]] + %0 = spirv.ShiftLeftLogical %arg0, %c32 : i32, i32 + %1 = spirv.ShiftLeftLogical %arg1, %cv : vector<3xi32>, vector<3xi32> + + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @const_fold_scalar_lsl +func.func @const_fold_scalar_lsl() -> i32 { + %c1 = spirv.Constant 65535 : i32 // 0x0000 ffff + %c2 = spirv.Constant 17 : i32 + + // CHECK: %[[RET:.*]] = spirv.Constant -131072 + // 0x0000 ffff << 17 -> 0xfffe 0000 + %0 = spirv.ShiftLeftLogical %c1, %c2 : i32, i32 + + // CHECK: return %[[RET]] + return %0 : i32 +} + +// CHECK-LABEL: @const_fold_vector_lsl +func.func @const_fold_vector_lsl() -> vector<3xi32> { + %c1 = spirv.Constant dense<[1, -1, 127]> : vector<3xi32> + %c2 = spirv.Constant dense<[31, 16, 13]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[-2147483648, -65536, 1040384]> + %0 = spirv.ShiftLeftLogical %c1, %c2 : vector<3xi32>, vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi32> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.RightShiftArithmetic +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @asr_x_0 +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @asr_x_0(%arg0 : i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %c0 = spirv.Constant 0 : i32 + %cv0 = spirv.Constant dense<0> : vector<3xi32> + + %0 = spirv.ShiftRightArithmetic %arg0, %c0 : i32, i32 + %1 = spirv.ShiftRightArithmetic %arg1, %cv0 : vector<3xi32>, vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @asr_shift_overflow +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @asr_shift_overflow(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + // CHECK-DAG: %[[C32:.*]] = spirv.Constant 32 + // CHECK-DAG: %[[CV:.*]] = spirv.Constant dense<[6, 18, 128]> + %c32 = spirv.Constant 32 : i32 + %cv = spirv.Constant dense<[6, 18, 128]> : vector<3xi32> + + // CHECK: %0 = spirv.ShiftRightArithmetic %[[ARG0]], %[[C32]] + // CHECK: %1 = spirv.ShiftRightArithmetic %[[ARG1]], %[[CV]] + %0 = spirv.ShiftRightArithmetic %arg0, %c32 : i32, i32 + %1 = spirv.ShiftRightArithmetic %arg1, %cv : vector<3xi32>, vector<3xi32> + + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @const_fold_scalar_asr +func.func @const_fold_scalar_asr() -> i32 { + %c1 = spirv.Constant -131072 : i32 // 0xfffe 0000 + %c2 = spirv.Constant 17 : i32 + // 0x0000 ffff ashr 17 -> 0xffff ffff + // CHECK: %[[RET:.*]] = spirv.Constant -1 + %0 = spirv.ShiftRightArithmetic %c1, %c2 : i32, i32 + + // CHECK: return %[[RET]] + return %0 : i32 +} + +// CHECK-LABEL: @const_fold_vector_asr +func.func @const_fold_vector_asr() -> vector<3xi32> { + %c1 = spirv.Constant dense<[-2147483648, 239847, 127]> : vector<3xi32> + %c2 = spirv.Constant dense<[31, 16, 13]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[-1, 3, 0]> + %0 = spirv.ShiftRightArithmetic %c1, %c2 : vector<3xi32>, vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi32> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.RightShiftLogical +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @lsr_x_0 +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @lsr_x_0(%arg0 : i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %c0 = spirv.Constant 0 : i32 + %cv0 = spirv.Constant dense<0> : vector<3xi32> + + %0 = spirv.ShiftRightLogical %arg0, %c0 : i32, i32 + %1 = spirv.ShiftRightLogical %arg1, %cv0 : vector<3xi32>, vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @lsr_shift_overflow +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @lsr_shift_overflow(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + // CHECK-DAG: %[[C32:.*]] = spirv.Constant 32 + // CHECK-DAG: %[[CV:.*]] = spirv.Constant dense<[6, 18, 128]> + %c32 = spirv.Constant 32 : i32 + %cv = spirv.Constant dense<[6, 18, 128]> : vector<3xi32> + + // CHECK: %0 = spirv.ShiftRightLogical %[[ARG0]], %[[C32]] + // CHECK: %1 = spirv.ShiftRightLogical %[[ARG1]], %[[CV]] + %0 = spirv.ShiftRightLogical %arg0, %c32 : i32, i32 + %1 = spirv.ShiftRightLogical %arg1, %cv : vector<3xi32>, vector<3xi32> + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @const_fold_scalar_lsr +func.func @const_fold_scalar_lsr() -> i32 { + %c1 = spirv.Constant -131072 : i32 // 0xfffe 0000 + %c2 = spirv.Constant 17 : i32 + + // 0x0000 ffff << 17 -> 0x0000 7fff + // CHECK: %[[RET:.*]] = spirv.Constant 32767 + %0 = spirv.ShiftRightLogical %c1, %c2 : i32, i32 + + // CHECK: return %[[RET]] + return %0 : i32 +} + +// CHECK-LABEL: @const_fold_vector_lsr +func.func @const_fold_vector_lsr() -> vector<3xi32> { + %c1 = spirv.Constant dense<[-2147483648, -1, -127]> : vector<3xi32> + %c2 = spirv.Constant dense<[31, 16, 13]> : vector<3xi32> + + // CHECK: %[[RET:.*]] = spirv.Constant dense<[1, 65535, 524287]> + %0 = spirv.ShiftRightLogical %c1, %c2 : vector<3xi32>, vector<3xi32> + + // CHECK: return %[[RET]] + return %0 : vector<3xi32> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.BitwiseAnd +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @bitwise_and_x_x +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @bitwise_and_x_x(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %0 = spirv.BitwiseAnd %arg0, %arg0 : i32 + %1 = spirv.BitwiseAnd %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @bitwise_and_x_0 +func.func @bitwise_and_x_0(%arg0 : i32, %arg1 : vector<3xi32>) -> (i32, vector<3xi32>) { + // CHECK-DAG: %[[C0:.*]] = spirv.Constant 0 : i32 + // CHECK-DAG: %[[CV0:.*]] = spirv.Constant dense<0> : vector<3xi32> + %c0 = spirv.Constant 0 : i32 + %cv0 = spirv.Constant dense<0> : vector<3xi32> + + %0 = spirv.BitwiseAnd %arg0, %c0 : i32 + %1 = spirv.BitwiseAnd %arg1, %cv0 : vector<3xi32> + + // CHECK: return %[[C0]], %[[CV0]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @bitwise_and_x_n1 +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @bitwise_and_x_n1(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %cn1 = spirv.Constant -1 : i32 + %cvn1 = spirv.Constant dense<-1> : vector<3xi32> + %0 = spirv.BitwiseAnd %arg0, %cn1 : i32 + %1 = spirv.BitwiseAnd %arg1, %cvn1 : vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @const_fold_scalar_band +func.func @const_fold_scalar_band() -> i32 { + %c1 = spirv.Constant -268464129 : i32 // 0xefff 8fff + %c2 = spirv.Constant 268464128: i32 // 0x1000 7000 + + // 0xefff 8fff | 0x1000 7000 = 0xffff ffff = -1 + // CHECK: %[[C0:.*]] = spirv.Constant 0 + %0 = spirv.BitwiseAnd %c1, %c2 : i32 + + // CHECK: return %[[C0]] + return %0 : i32 +} + +// CHECK-LABEL: @const_fold_vector_band +func.func @const_fold_vector_band() -> vector<3xi32> { + %c1 = spirv.Constant dense<[42, -55, 127]> : vector<3xi32> + %c2 = spirv.Constant dense<[-3, -15, 28]> : vector<3xi32> + + // CHECK: %[[CV:.*]] = spirv.Constant dense<[40, -63, 28]> + %0 = spirv.BitwiseAnd %c1, %c2 : vector<3xi32> + + // CHECK: return %[[CV]] + return %0 : vector<3xi32> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.BitwiseOr +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @bitwise_or_x_x +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @bitwise_or_x_x(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %0 = spirv.BitwiseOr %arg0, %arg0 : i32 + %1 = spirv.BitwiseOr %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @bitwise_or_x_0 +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @bitwise_or_x_0(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %c1 = spirv.Constant 0 : i32 + %cv1 = spirv.Constant dense<0> : vector<3xi32> + %0 = spirv.BitwiseOr %arg0, %c1 : i32 + %1 = spirv.BitwiseOr %arg1, %cv1 : vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @bitwise_or_x_n1 +func.func @bitwise_or_x_n1(%arg0 : i32, %arg1 : vector<3xi32>) -> (i32, vector<3xi32>) { + // CHECK-DAG: %[[CN1:.*]] = spirv.Constant -1 : i32 + // CHECK-DAG: %[[CVN1:.*]] = spirv.Constant dense<-1> : vector<3xi32> + %cn1 = spirv.Constant -1 : i32 + %cvn1 = spirv.Constant dense<-1> : vector<3xi32> + %0 = spirv.BitwiseOr %arg0, %cn1 : i32 + %1 = spirv.BitwiseOr %arg1, %cvn1 : vector<3xi32> + + // CHECK: return %[[CN1]], %[[CVN1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @const_fold_scalar_bor +func.func @const_fold_scalar_bor() -> i32 { + %c1 = spirv.Constant -268464129 : i32 // 0xefff 8fff + %c2 = spirv.Constant 268464128: i32 // 0x1000 7000 + + // 0xefff 8fff | 0x1000 7000 = 0xffff ffff = -1 + // CHECK: %[[CN1:.*]] = spirv.Constant -1 + %0 = spirv.BitwiseOr %c1, %c2 : i32 + + // CHECK: return %[[CN1]] + return %0 : i32 +} + +// CHECK-LABEL: @const_fold_vector_bor +func.func @const_fold_vector_bor() -> vector<3xi32> { + %c1 = spirv.Constant dense<[42, -55, 127]> : vector<3xi32> + %c2 = spirv.Constant dense<[-3, -15, 28]> : vector<3xi32> + + // CHECK: %[[CV:.*]] = spirv.Constant dense<[-1, -7, 127]> + %0 = spirv.BitwiseOr %c1, %c2 : vector<3xi32> + + // CHECK: return %[[CV]] + return %0 : vector<3xi32> +} + +// ----- + +//===----------------------------------------------------------------------===// +// spirv.BitwiseXor +//===----------------------------------------------------------------------===// + +// CHECK-LABEL: @bitwise_xor_x_0 +// CHECK-SAME: (%[[ARG0:.*]]: i32, %[[ARG1:.*]]: vector<3xi32>) +func.func @bitwise_xor_x_0(%arg0: i32, %arg1: vector<3xi32>) -> (i32, vector<3xi32>) { + %c0 = spirv.Constant 0 : i32 + %cv0 = spirv.Constant dense<0> : vector<3xi32> + + %0 = spirv.BitwiseXor %arg0, %c0 : i32 + %1 = spirv.BitwiseXor %arg1, %cv0 : vector<3xi32> + + // CHECK: return %[[ARG0]], %[[ARG1]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @bitwise_xor_x_x +func.func @bitwise_xor_x_x(%arg0 : i32, %arg1 : vector<3xi32>) -> (i32, vector<3xi32>) { + // CHECK-DAG: %[[C0:.*]] = spirv.Constant 0 + // CHECK-DAG: %[[CV0:.*]] = spirv.Constant dense<0> + %0 = spirv.BitwiseXor %arg0, %arg0 : i32 + %1 = spirv.BitwiseXor %arg1, %arg1 : vector<3xi32> + + // CHECK: return %[[C0]], %[[CV0]] + return %0, %1 : i32, vector<3xi32> +} + +// CHECK-LABEL: @const_fold_scalar_bxor +func.func @const_fold_scalar_bxor() -> i32 { + %c1 = spirv.Constant 4294967295 : i32 // 2^32 - 1: 0xffff ffff + %c2 = spirv.Constant -2147483648 : i32 // -2^31 : 0x8000 0000 + + // 0x8000 0000 ^ 0xffff fffe = 0xefff ffff + // CHECK: %[[CBIG:.*]] = spirv.Constant 2147483647 + %0 = spirv.BitwiseXor %c1, %c2 : i32 + + // CHECK: return %[[CBIG]] + return %0 : i32 +} + +// CHECK-LABEL: @const_fold_vector_bxor +func.func @const_fold_vector_bxor() -> vector<3xi32> { + %c1 = spirv.Constant dense<[42, -55, 127]> : vector<3xi32> + %c2 = spirv.Constant dense<[-3, -15, 28]> : vector<3xi32> + + // CHECK: %[[CV:.*]] = spirv.Constant dense<[-41, 56, 99]> + %0 = spirv.BitwiseXor %c1, %c2 : vector<3xi32> + + // CHECK: return %[[CV]] + return %0 : vector<3xi32> +} + +// ----- + //===----------------------------------------------------------------------===// // spirv.mlir.selection //===----------------------------------------------------------------------===// diff --git a/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir b/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir index 0f99a0206e4c..a3c1e76a3d09 100644 --- a/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir +++ b/mlir/test/Dialect/SparseTensor/sparse_conv_2d_slice_based.mlir @@ -8,263 +8,230 @@ // CHECK-LABEL: func.func @conv2d_all_sparse_CSR( -// CHECK-SAME: %[[VAL_0:.*]]: tensor<8x8xi32, #sparse{{[0-9]*}}>, -// CHECK-SAME: %[[VAL_1:.*]]: tensor<3x3xi32>) -> tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK-SAME: %[[VAL_0:.*]]: tensor<8x8xi32, #sparse>, +// CHECK-SAME: %[[VAL_1:.*]]: tensor<3x3xi32>) -> tensor<6x6xi32, #sparse> { // CHECK-DAG: %[[VAL_2:.*]] = arith.constant true // CHECK-DAG: %[[VAL_3:.*]] = arith.constant -2 : index -// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 4 : index +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 2 : index // CHECK-DAG: %[[VAL_5:.*]] = arith.constant 8 : index // CHECK-DAG: %[[VAL_6:.*]] = arith.constant 3 : index // CHECK-DAG: %[[VAL_7:.*]] = arith.constant 1 : index -// CHECK-DAG: %[[VAL_8:.*]] = arith.constant 5 : index -// CHECK-DAG: %[[VAL_9:.*]] = arith.constant 2 : index -// CHECK-DAG: %[[VAL_10:.*]] = arith.constant 0 : index -// CHECK-DAG: %[[VAL_11:.*]] = arith.constant 0 : i32 -// CHECK-DAG: %[[VAL_12:.*]] = arith.constant false -// CHECK-DAG: %[[VAL_13:.*]] = tensor.empty() : tensor<6x6xi32, #sparse{{[0-9]*}}> -// CHECK-DAG: %[[VAL_14:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 0 : index} : tensor<8x8xi32, #sparse{{[0-9]*}}> to memref -// CHECK-DAG: %[[VAL_15:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 0 : index} : tensor<8x8xi32, #sparse{{[0-9]*}}> to memref -// CHECK-DAG: %[[VAL_16:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 1 : index} : tensor<8x8xi32, #sparse{{[0-9]*}}> to memref -// CHECK-DAG: %[[VAL_17:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 1 : index} : tensor<8x8xi32, #sparse{{[0-9]*}}> to memref -// CHECK-DAG: %[[VAL_18:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<8x8xi32, #sparse{{[0-9]*}}> to memref -// CHECK-DAG: %[[VAL_19:.*]] = memref.alloca() : memref<11xindex> -// CHECK-DAG: %[[VAL_20:.*]] = memref.alloca() : memref<5xindex> -// CHECK-DAG: %[[VAL_21:.*]] = memref.load %[[VAL_14]]{{\[}}%[[VAL_7]]] : memref -// CHECK-DAG: memref.store %[[VAL_7]], %[[VAL_20]]{{\[}}%[[VAL_10]]] : memref<5xindex> -// CHECK-DAG: memref.store %[[VAL_10]], %[[VAL_20]]{{\[}}%[[VAL_7]]] : memref<5xindex> -// CHECK-DAG: memref.store %[[VAL_10]], %[[VAL_20]]{{\[}}%[[VAL_9]]] : memref<5xindex> -// CHECK-DAG: memref.store %[[VAL_21]], %[[VAL_20]]{{\[}}%[[VAL_6]]] : memref<5xindex> -// CHECK: %[[VAL_22:.*]] = arith.cmpi ugt, %[[VAL_21]], %[[VAL_10]] : index -// CHECK: %[[VAL_23:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_10]]] : memref -// CHECK: %[[VAL_24:.*]] = arith.cmpi uge, %[[VAL_23]], %[[VAL_6]] : index -// CHECK: %[[VAL_25:.*]] = arith.andi %[[VAL_22]], %[[VAL_24]] : i1 -// CHECK: %[[VAL_26:.*]] = arith.addi %[[VAL_23]], %[[VAL_3]] : index -// CHECK: %[[VAL_27:.*]] = arith.select %[[VAL_25]], %[[VAL_26]], %[[VAL_10]] : index -// CHECK: %[[VAL_28:.*]]:3 = scf.while (%[[VAL_29:.*]] = %[[VAL_22]], %[[VAL_30:.*]] = %[[VAL_23]], %[[VAL_31:.*]] = %[[VAL_27]], %[[VAL_32:.*]] = %[[VAL_13]]) : (i1, index, index, tensor<6x6xi32, #sparse{{[0-9]*}}>) -> (index, index, tensor<6x6xi32, #sparse{{[0-9]*}}>) { -// CHECK: scf.condition(%[[VAL_29]]) %[[VAL_30]], %[[VAL_31]], %[[VAL_32]] : index, index, tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK-DAG: %[[VAL_8:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_9:.*]] = arith.constant 0 : i32 +// CHECK-DAG: %[[VAL_10:.*]] = arith.constant false +// CHECK-DAG: %[[VAL_11:.*]] = tensor.empty() : tensor<6x6xi32, #sparse> +// CHECK-DAG: %[[VAL_12:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 0 : index} : tensor<8x8xi32, #sparse> to memref +// CHECK-DAG: %[[VAL_13:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 0 : index} : tensor<8x8xi32, #sparse> to memref +// CHECK-DAG: %[[VAL_14:.*]] = sparse_tensor.positions %[[VAL_0]] {level = 1 : index} : tensor<8x8xi32, #sparse> to memref +// CHECK-DAG: %[[VAL_15:.*]] = sparse_tensor.coordinates %[[VAL_0]] {level = 1 : index} : tensor<8x8xi32, #sparse> to memref +// CHECK-DAG: %[[VAL_16:.*]] = sparse_tensor.values %[[VAL_0]] : tensor<8x8xi32, #sparse> to memref +// CHECK-DAG: %[[VAL_17:.*]] = memref.alloca() : memref<9xindex> +// CHECK-DAG: %[[VAL_18:.*]] = memref.alloca() : memref<3xindex> +// CHECK-DAG: %[[VAL_19:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_7]]] : memref +// CHECK: memref.store %[[VAL_8]], %[[VAL_18]]{{\[}}%[[VAL_8]]] : memref<3xindex> +// CHECK: memref.store %[[VAL_19]], %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<3xindex> +// CHECK: %[[VAL_20:.*]] = arith.cmpi ugt, %[[VAL_19]], %[[VAL_8]] : index +// CHECK: %[[VAL_21:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_8]]] : memref +// CHECK: %[[VAL_22:.*]] = arith.cmpi uge, %[[VAL_21]], %[[VAL_6]] : index +// CHECK: %[[VAL_23:.*]] = arith.andi %[[VAL_20]], %[[VAL_22]] : i1 +// CHECK: %[[VAL_24:.*]] = arith.addi %[[VAL_21]], %[[VAL_3]] : index +// CHECK: %[[VAL_25:.*]] = arith.select %[[VAL_23]], %[[VAL_24]], %[[VAL_8]] : index +// CHECK: %[[VAL_26:.*]]:3 = scf.while (%[[VAL_27:.*]] = %[[VAL_20]], %[[VAL_28:.*]] = %[[VAL_21]], %[[VAL_29:.*]] = %[[VAL_25]], %[[VAL_30:.*]] = %[[VAL_11]]) : (i1, index, index, tensor<6x6xi32, #sparse>) -> (index, index, tensor<6x6xi32, #sparse>) { +// CHECK: scf.condition(%[[VAL_27]]) %[[VAL_28]], %[[VAL_29]], %[[VAL_30]] : index, index, tensor<6x6xi32, #sparse> // CHECK: } do { -// CHECK: ^bb0(%[[VAL_33:.*]]: index, %[[VAL_34:.*]]: index, %[[VAL_35:.*]]: tensor<6x6xi32, #sparse{{[0-9]*}}>): -// CHECK: %[[VAL_36:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_10]]] : memref<5xindex> -// CHECK: %[[VAL_37:.*]]:4 = scf.for %[[VAL_38:.*]] = %[[VAL_10]] to %[[VAL_36]] step %[[VAL_7]] iter_args(%[[VAL_39:.*]] = %[[VAL_12]], %[[VAL_40:.*]] = %[[VAL_5]], %[[VAL_41:.*]] = %[[VAL_10]], %[[VAL_42:.*]] = %[[VAL_10]]) -> (i1, index, index, index) { -// CHECK: %[[VAL_43:.*]] = arith.addi %[[VAL_38]], %[[VAL_9]] : index -// CHECK: %[[VAL_44:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_43]]] : memref<5xindex> -// CHECK: %[[VAL_45:.*]] = arith.addi %[[VAL_38]], %[[VAL_6]] : index -// CHECK: %[[VAL_46:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_45]]] : memref<5xindex> -// CHECK: %[[VAL_47:.*]] = arith.addi %[[VAL_38]], %[[VAL_4]] : index -// CHECK: memref.store %[[VAL_42]], %[[VAL_20]]{{\[}}%[[VAL_47]]] : memref<5xindex> -// CHECK: %[[VAL_48:.*]] = arith.addi %[[VAL_34]], %[[VAL_6]] : index -// CHECK: %[[VAL_49:.*]]:5 = scf.while (%[[VAL_50:.*]] = %[[VAL_44]], %[[VAL_51:.*]] = %[[VAL_39]], %[[VAL_52:.*]] = %[[VAL_40]], %[[VAL_53:.*]] = %[[VAL_41]], %[[VAL_54:.*]] = %[[VAL_42]]) : (index, i1, index, index, index) -> (index, i1, index, index, index) { -// CHECK: %[[VAL_55:.*]] = arith.cmpi ult, %[[VAL_50]], %[[VAL_46]] : index -// CHECK: %[[VAL_56:.*]] = scf.if %[[VAL_55]] -> (i1) { -// CHECK: %[[VAL_57:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_50]]] : memref -// CHECK: %[[VAL_58:.*]] = arith.cmpi ult, %[[VAL_57]], %[[VAL_48]] : index -// CHECK: scf.yield %[[VAL_58]] : i1 -// CHECK: } else { -// CHECK: scf.yield %[[VAL_12]] : i1 -// CHECK: } -// CHECK: scf.condition(%[[VAL_56]]) %[[VAL_50]], %[[VAL_51]], %[[VAL_52]], %[[VAL_53]], %[[VAL_54]] : index, i1, index, index, index -// CHECK: } do { -// CHECK: ^bb0(%[[VAL_59:.*]]: index, %[[VAL_60:.*]]: i1, %[[VAL_61:.*]]: index, %[[VAL_62:.*]]: index, %[[VAL_63:.*]]: index): -// CHECK: %[[VAL_64:.*]] = arith.addi %[[VAL_59]], %[[VAL_7]] : index -// CHECK: %[[VAL_65:.*]] = memref.load %[[VAL_16]]{{\[}}%[[VAL_59]]] : memref -// CHECK: %[[VAL_66:.*]] = memref.load %[[VAL_16]]{{\[}}%[[VAL_64]]] : memref -// CHECK: %[[VAL_67:.*]] = arith.cmpi ult, %[[VAL_65]], %[[VAL_66]] : index -// CHECK: %[[VAL_68:.*]] = arith.ori %[[VAL_67]], %[[VAL_60]] : i1 -// CHECK: %[[VAL_69:.*]] = scf.if %[[VAL_67]] -> (index) { -// CHECK: %[[VAL_70:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_65]]] : memref -// CHECK: %[[VAL_71:.*]] = arith.cmpi ult, %[[VAL_70]], %[[VAL_61]] : index -// CHECK: %[[VAL_72:.*]] = arith.select %[[VAL_71]], %[[VAL_70]], %[[VAL_61]] : index -// CHECK: scf.yield %[[VAL_72]] : index -// CHECK: } else { -// CHECK: scf.yield %[[VAL_61]] : index -// CHECK: } -// CHECK: %[[VAL_73:.*]] = arith.addi %[[VAL_62]], %[[VAL_9]] : index -// CHECK: memref.store %[[VAL_65]], %[[VAL_19]]{{\[}}%[[VAL_73]]] : memref<11xindex> -// CHECK: %[[VAL_74:.*]] = arith.addi %[[VAL_62]], %[[VAL_8]] : index -// CHECK: memref.store %[[VAL_66]], %[[VAL_19]]{{\[}}%[[VAL_74]]] : memref<11xindex> -// CHECK: %[[VAL_75:.*]] = arith.addi %[[VAL_62]], %[[VAL_7]] : index -// CHECK: %[[VAL_76:.*]] = arith.addi %[[VAL_63]], %[[VAL_7]] : index -// CHECK: scf.yield %[[VAL_64]], %[[VAL_68]], %[[VAL_69]], %[[VAL_75]], %[[VAL_76]] : index, i1, index, index, index +// CHECK: ^bb0(%[[VAL_31:.*]]: index, %[[VAL_32:.*]]: index, %[[VAL_33:.*]]: tensor<6x6xi32, #sparse>): +// CHECK: %[[VAL_34:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_8]]] : memref<3xindex> +// CHECK: %[[VAL_35:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<3xindex> +// CHECK: memref.store %[[VAL_8]], %[[VAL_18]]{{\[}}%[[VAL_4]]] : memref<3xindex> +// CHECK: %[[VAL_36:.*]] = arith.addi %[[VAL_32]], %[[VAL_6]] : index +// CHECK: %[[VAL_37:.*]]:5 = scf.while (%[[VAL_38:.*]] = %[[VAL_34]], %[[VAL_39:.*]] = %[[VAL_10]], %[[VAL_40:.*]] = %[[VAL_5]], %[[VAL_41:.*]] = %[[VAL_8]], %[[VAL_42:.*]] = %[[VAL_8]]) : (index, i1, index, index, index) -> (index, i1, index, index, index) { +// CHECK: %[[VAL_43:.*]] = arith.cmpi ult, %[[VAL_38]], %[[VAL_35]] : index +// CHECK: %[[VAL_44:.*]] = scf.if %[[VAL_43]] -> (i1) { +// CHECK: %[[VAL_45:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_38]]] : memref +// CHECK: %[[VAL_46:.*]] = arith.cmpi ult, %[[VAL_45]], %[[VAL_36]] : index +// CHECK: scf.yield %[[VAL_46]] : i1 +// CHECK: } else { +// CHECK: scf.yield %[[VAL_10]] : i1 +// CHECK: } +// CHECK: scf.condition(%[[VAL_44]]) %[[VAL_38]], %[[VAL_39]], %[[VAL_40]], %[[VAL_41]], %[[VAL_42]] : index, i1, index, index, index +// CHECK: } do { +// CHECK: ^bb0(%[[VAL_47:.*]]: index, %[[VAL_48:.*]]: i1, %[[VAL_49:.*]]: index, %[[VAL_50:.*]]: index, %[[VAL_51:.*]]: index): +// CHECK: %[[VAL_52:.*]] = arith.addi %[[VAL_47]], %[[VAL_7]] : index +// CHECK: %[[VAL_53:.*]] = memref.load %[[VAL_14]]{{\[}}%[[VAL_47]]] : memref +// CHECK: %[[VAL_54:.*]] = memref.load %[[VAL_14]]{{\[}}%[[VAL_52]]] : memref +// CHECK: %[[VAL_55:.*]] = arith.cmpi ult, %[[VAL_53]], %[[VAL_54]] : index +// CHECK: %[[VAL_56:.*]] = arith.ori %[[VAL_55]], %[[VAL_48]] : i1 +// CHECK: %[[VAL_57:.*]] = scf.if %[[VAL_55]] -> (index) { +// CHECK: %[[VAL_58:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_53]]] : memref +// CHECK: %[[VAL_59:.*]] = arith.cmpi ult, %[[VAL_58]], %[[VAL_49]] : index +// CHECK: %[[VAL_60:.*]] = arith.select %[[VAL_59]], %[[VAL_58]], %[[VAL_49]] : index +// CHECK: scf.yield %[[VAL_60]] : index +// CHECK: } else { +// CHECK: scf.yield %[[VAL_49]] : index // CHECK: } -// CHECK: scf.yield %[[VAL_77:.*]]#1, %[[VAL_77]]#2, %[[VAL_77]]#3, %[[VAL_77]]#4 : i1, index, index, index +// CHECK: memref.store %[[VAL_53]], %[[VAL_17]]{{\[}}%[[VAL_50]]] : memref<9xindex> +// CHECK: %[[VAL_61:.*]] = arith.addi %[[VAL_50]], %[[VAL_6]] : index +// CHECK: memref.store %[[VAL_54]], %[[VAL_17]]{{\[}}%[[VAL_61]]] : memref<9xindex> +// CHECK: %[[VAL_62:.*]] = arith.addi %[[VAL_50]], %[[VAL_7]] : index +// CHECK: %[[VAL_63:.*]] = arith.addi %[[VAL_51]], %[[VAL_7]] : index +// CHECK: scf.yield %[[VAL_52]], %[[VAL_56]], %[[VAL_57]], %[[VAL_62]], %[[VAL_63]] : index, i1, index, index, index // CHECK: } -// CHECK: memref.store %[[VAL_78:.*]]#2, %[[VAL_19]]{{\[}}%[[VAL_10]]] : memref<11xindex> -// CHECK: memref.store %[[VAL_10]], %[[VAL_19]]{{\[}}%[[VAL_7]]] : memref<11xindex> -// CHECK: %[[VAL_79:.*]] = arith.cmpi uge, %[[VAL_78]]#1, %[[VAL_6]] : index -// CHECK: %[[VAL_80:.*]] = arith.andi %[[VAL_78]]#0, %[[VAL_79]] : i1 -// CHECK: %[[VAL_81:.*]] = arith.addi %[[VAL_78]]#1, %[[VAL_3]] : index -// CHECK: %[[VAL_82:.*]] = arith.select %[[VAL_80]], %[[VAL_81]], %[[VAL_10]] : index -// CHECK: %[[VAL_83:.*]]:3 = scf.while (%[[VAL_84:.*]] = %[[VAL_78]]#0, %[[VAL_85:.*]] = %[[VAL_78]]#1, %[[VAL_86:.*]] = %[[VAL_82]], %[[VAL_87:.*]] = %[[VAL_35]]) : (i1, index, index, tensor<6x6xi32, #sparse{{[0-9]*}}>) -> (index, index, tensor<6x6xi32, #sparse{{[0-9]*}}>) { -// CHECK: scf.condition(%[[VAL_84]]) %[[VAL_85]], %[[VAL_86]], %[[VAL_87]] : index, index, tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_64:.*]] = arith.cmpi uge, %[[VAL_65:.*]]#2, %[[VAL_6]] : index +// CHECK: %[[VAL_66:.*]] = arith.andi %[[VAL_65]]#1, %[[VAL_64]] : i1 +// CHECK: %[[VAL_67:.*]] = arith.addi %[[VAL_65]]#2, %[[VAL_3]] : index +// CHECK: %[[VAL_68:.*]] = arith.select %[[VAL_66]], %[[VAL_67]], %[[VAL_8]] : index +// CHECK: %[[VAL_69:.*]]:3 = scf.while (%[[VAL_70:.*]] = %[[VAL_65]]#1, %[[VAL_71:.*]] = %[[VAL_65]]#2, %[[VAL_72:.*]] = %[[VAL_68]], %[[VAL_73:.*]] = %[[VAL_33]]) : (i1, index, index, tensor<6x6xi32, #sparse>) -> (index, index, tensor<6x6xi32, #sparse>) { +// CHECK: scf.condition(%[[VAL_70]]) %[[VAL_71]], %[[VAL_72]], %[[VAL_73]] : index, index, tensor<6x6xi32, #sparse> // CHECK: } do { -// CHECK: ^bb0(%[[VAL_88:.*]]: index, %[[VAL_89:.*]]: index, %[[VAL_90:.*]]: tensor<6x6xi32, #sparse{{[0-9]*}}>): -// CHECK: %[[VAL_91:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_7]]] : memref<5xindex> -// CHECK: %[[VAL_92:.*]] = arith.addi %[[VAL_91]], %[[VAL_9]] : index -// CHECK: %[[VAL_93:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_92]]] : memref<5xindex> -// CHECK: %[[VAL_94:.*]] = arith.addi %[[VAL_91]], %[[VAL_6]] : index -// CHECK: %[[VAL_95:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_94]]] : memref<5xindex> -// CHECK: %[[VAL_96:.*]] = arith.addi %[[VAL_34]], %[[VAL_6]] : index -// CHECK: %[[VAL_97:.*]]:3 = scf.while (%[[VAL_98:.*]] = %[[VAL_93]], %[[VAL_99:.*]] = %[[VAL_11]], %[[VAL_100:.*]] = %[[VAL_12]]) : (index, i32, i1) -> (index, i32, i1) { -// CHECK: %[[VAL_101:.*]] = arith.cmpi ult, %[[VAL_98]], %[[VAL_95]] : index -// CHECK: %[[VAL_102:.*]] = scf.if %[[VAL_101]] -> (i1) { -// CHECK: %[[VAL_103:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_98]]] : memref -// CHECK: %[[VAL_104:.*]] = arith.cmpi ult, %[[VAL_103]], %[[VAL_96]] : index -// CHECK: scf.yield %[[VAL_104]] : i1 +// CHECK: ^bb0(%[[VAL_74:.*]]: index, %[[VAL_75:.*]]: index, %[[VAL_76:.*]]: tensor<6x6xi32, #sparse>): +// CHECK: %[[VAL_77:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_8]]] : memref<3xindex> +// CHECK: %[[VAL_78:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<3xindex> +// CHECK: %[[VAL_79:.*]]:3 = scf.while (%[[VAL_80:.*]] = %[[VAL_77]], %[[VAL_81:.*]] = %[[VAL_9]], %[[VAL_82:.*]] = %[[VAL_10]]) : (index, i32, i1) -> (index, i32, i1) { +// CHECK: %[[VAL_83:.*]] = arith.cmpi ult, %[[VAL_80]], %[[VAL_78]] : index +// CHECK: %[[VAL_84:.*]] = scf.if %[[VAL_83]] -> (i1) { +// CHECK: %[[VAL_85:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_80]]] : memref +// CHECK: %[[VAL_86:.*]] = arith.cmpi ult, %[[VAL_85]], %[[VAL_36]] : index +// CHECK: scf.yield %[[VAL_86]] : i1 // CHECK: } else { -// CHECK: scf.yield %[[VAL_12]] : i1 +// CHECK: scf.yield %[[VAL_10]] : i1 // CHECK: } -// CHECK: scf.condition(%[[VAL_102]]) %[[VAL_98]], %[[VAL_99]], %[[VAL_100]] : index, i32, i1 +// CHECK: scf.condition(%[[VAL_84]]) %[[VAL_80]], %[[VAL_81]], %[[VAL_82]] : index, i32, i1 // CHECK: } do { -// CHECK: ^bb0(%[[VAL_105:.*]]: index, %[[VAL_106:.*]]: i32, %[[VAL_107:.*]]: i1): -// CHECK: %[[VAL_108:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_105]]] : memref -// CHECK: %[[VAL_109:.*]] = arith.subi %[[VAL_108]], %[[VAL_34]] : index -// CHECK: %[[VAL_110:.*]] = memref.load %[[VAL_19]]{{\[}}%[[VAL_7]]] : memref<11xindex> -// CHECK: %[[VAL_111:.*]] = arith.addi %[[VAL_110]], %[[VAL_9]] : index -// CHECK: %[[VAL_112:.*]] = memref.load %[[VAL_19]]{{\[}}%[[VAL_111]]] : memref<11xindex> -// CHECK: %[[VAL_113:.*]] = arith.addi %[[VAL_110]], %[[VAL_8]] : index -// CHECK: %[[VAL_114:.*]] = memref.load %[[VAL_19]]{{\[}}%[[VAL_113]]] : memref<11xindex> -// CHECK: %[[VAL_115:.*]] = arith.addi %[[VAL_89]], %[[VAL_6]] : index -// CHECK: %[[VAL_116:.*]]:2 = scf.while (%[[VAL_117:.*]] = %[[VAL_112]], %[[VAL_118:.*]] = %[[VAL_106]]) : (index, i32) -> (index, i32) { -// CHECK: %[[VAL_119:.*]] = arith.cmpi ult, %[[VAL_117]], %[[VAL_114]] : index -// CHECK: %[[VAL_120:.*]] = scf.if %[[VAL_119]] -> (i1) { -// CHECK: %[[VAL_121:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_117]]] : memref -// CHECK: %[[VAL_122:.*]] = arith.cmpi ult, %[[VAL_121]], %[[VAL_115]] : index -// CHECK: scf.yield %[[VAL_122]] : i1 +// CHECK: ^bb0(%[[VAL_87:.*]]: index, %[[VAL_88:.*]]: i32, %[[VAL_89:.*]]: i1): +// CHECK: %[[VAL_90:.*]] = arith.subi %[[VAL_87]], %[[VAL_77]] : index +// CHECK: %[[VAL_91:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_87]]] : memref +// CHECK: %[[VAL_92:.*]] = arith.subi %[[VAL_91]], %[[VAL_32]] : index +// CHECK: %[[VAL_93:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_90]]] : memref<9xindex> +// CHECK: %[[VAL_94:.*]] = arith.addi %[[VAL_90]], %[[VAL_6]] : index +// CHECK: %[[VAL_95:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_94]]] : memref<9xindex> +// CHECK: %[[VAL_96:.*]] = arith.addi %[[VAL_75]], %[[VAL_6]] : index +// CHECK: %[[VAL_97:.*]]:2 = scf.while (%[[VAL_98:.*]] = %[[VAL_93]], %[[VAL_99:.*]] = %[[VAL_88]]) : (index, i32) -> (index, i32) { +// CHECK: %[[VAL_100:.*]] = arith.cmpi ult, %[[VAL_98]], %[[VAL_95]] : index +// CHECK: %[[VAL_101:.*]] = scf.if %[[VAL_100]] -> (i1) { +// CHECK: %[[VAL_102:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_98]]] : memref +// CHECK: %[[VAL_103:.*]] = arith.cmpi ult, %[[VAL_102]], %[[VAL_96]] : index +// CHECK: scf.yield %[[VAL_103]] : i1 // CHECK: } else { -// CHECK: scf.yield %[[VAL_12]] : i1 +// CHECK: scf.yield %[[VAL_10]] : i1 // CHECK: } -// CHECK: scf.condition(%[[VAL_120]]) %[[VAL_117]], %[[VAL_118]] : index, i32 +// CHECK: scf.condition(%[[VAL_101]]) %[[VAL_98]], %[[VAL_99]] : index, i32 // CHECK: } do { -// CHECK: ^bb0(%[[VAL_123:.*]]: index, %[[VAL_124:.*]]: i32): -// CHECK: %[[VAL_125:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_123]]] : memref -// CHECK: %[[VAL_126:.*]] = arith.subi %[[VAL_125]], %[[VAL_89]] : index -// CHECK: %[[VAL_127:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_123]]] : memref -// CHECK: %[[VAL_128:.*]] = tensor.extract %[[VAL_1]]{{\[}}%[[VAL_109]], %[[VAL_126]]] : tensor<3x3xi32> -// CHECK: %[[VAL_129:.*]] = arith.muli %[[VAL_127]], %[[VAL_128]] : i32 -// CHECK: %[[VAL_130:.*]] = arith.addi %[[VAL_124]], %[[VAL_129]] : i32 -// CHECK: %[[VAL_131:.*]] = arith.addi %[[VAL_123]], %[[VAL_7]] : index -// CHECK: scf.yield %[[VAL_131]], %[[VAL_130]] : index, i32 +// CHECK: ^bb0(%[[VAL_104:.*]]: index, %[[VAL_105:.*]]: i32): +// CHECK: %[[VAL_106:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_104]]] : memref +// CHECK: %[[VAL_107:.*]] = arith.subi %[[VAL_106]], %[[VAL_75]] : index +// CHECK: %[[VAL_108:.*]] = memref.load %[[VAL_16]]{{\[}}%[[VAL_104]]] : memref +// CHECK: %[[VAL_109:.*]] = tensor.extract %[[VAL_1]]{{\[}}%[[VAL_92]], %[[VAL_107]]] : tensor<3x3xi32> +// CHECK: %[[VAL_110:.*]] = arith.muli %[[VAL_108]], %[[VAL_109]] : i32 +// CHECK: %[[VAL_111:.*]] = arith.addi %[[VAL_105]], %[[VAL_110]] : i32 +// CHECK: %[[VAL_112:.*]] = arith.addi %[[VAL_104]], %[[VAL_7]] : index +// CHECK: scf.yield %[[VAL_112]], %[[VAL_111]] : index, i32 // CHECK: } -// CHECK: %[[VAL_132:.*]] = arith.addi %[[VAL_105]], %[[VAL_7]] : index -// CHECK: %[[VAL_133:.*]] = arith.addi %[[VAL_110]], %[[VAL_7]] : index -// CHECK: memref.store %[[VAL_133]], %[[VAL_19]]{{\[}}%[[VAL_7]]] : memref<11xindex> -// CHECK: scf.yield %[[VAL_132]], %[[VAL_134:.*]]#1, %[[VAL_2]] : index, i32, i1 +// CHECK: %[[VAL_113:.*]] = arith.addi %[[VAL_87]], %[[VAL_7]] : index +// CHECK: scf.yield %[[VAL_113]], %[[VAL_114:.*]]#1, %[[VAL_2]] : index, i32, i1 // CHECK: } -// CHECK: %[[VAL_135:.*]] = scf.if %[[VAL_136:.*]]#2 -> (tensor<6x6xi32, #sparse{{[0-9]*}}>) { -// CHECK: %[[VAL_137:.*]] = sparse_tensor.insert %[[VAL_136]]#1 into %[[VAL_90]]{{\[}}%[[VAL_34]], %[[VAL_89]]] : tensor<6x6xi32, #sparse{{[0-9]*}}> -// CHECK: scf.yield %[[VAL_137]] : tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_115:.*]] = scf.if %[[VAL_116:.*]]#2 -> (tensor<6x6xi32, #sparse>) { +// CHECK: %[[VAL_117:.*]] = sparse_tensor.insert %[[VAL_116]]#1 into %[[VAL_76]]{{\[}}%[[VAL_32]], %[[VAL_75]]] : tensor<6x6xi32, #sparse> +// CHECK: scf.yield %[[VAL_117]] : tensor<6x6xi32, #sparse> // CHECK: } else { -// CHECK: scf.yield %[[VAL_90]] : tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK: scf.yield %[[VAL_76]] : tensor<6x6xi32, #sparse> // CHECK: } -// CHECK: memref.store %[[VAL_10]], %[[VAL_20]]{{\[}}%[[VAL_7]]] : memref<5xindex> -// CHECK: memref.store %[[VAL_10]], %[[VAL_19]]{{\[}}%[[VAL_7]]] : memref<11xindex> -// CHECK: %[[VAL_138:.*]] = arith.cmpi ugt, %[[VAL_88]], %[[VAL_89]] : index -// CHECK: %[[VAL_139:.*]]:3 = scf.if %[[VAL_138]] -> (index, i1, index) { -// CHECK: %[[VAL_140:.*]] = arith.addi %[[VAL_89]], %[[VAL_7]] : index -// CHECK: scf.yield %[[VAL_88]], %[[VAL_2]], %[[VAL_140]] : index, i1, index +// CHECK: %[[VAL_118:.*]] = arith.cmpi ugt, %[[VAL_74]], %[[VAL_75]] : index +// CHECK: %[[VAL_119:.*]]:3 = scf.if %[[VAL_118]] -> (index, i1, index) { +// CHECK: %[[VAL_120:.*]] = arith.addi %[[VAL_75]], %[[VAL_7]] : index +// CHECK: scf.yield %[[VAL_74]], %[[VAL_2]], %[[VAL_120]] : index, i1, index // CHECK: } else { -// CHECK: %[[VAL_141:.*]] = memref.load %[[VAL_19]]{{\[}}%[[VAL_10]]] : memref<11xindex> -// CHECK: %[[VAL_142:.*]]:2 = scf.for %[[VAL_143:.*]] = %[[VAL_10]] to %[[VAL_141]] step %[[VAL_7]] iter_args(%[[VAL_144:.*]] = %[[VAL_5]], %[[VAL_145:.*]] = %[[VAL_12]]) -> (index, i1) { -// CHECK: %[[VAL_146:.*]] = arith.addi %[[VAL_143]], %[[VAL_9]] : index -// CHECK: %[[VAL_147:.*]] = memref.load %[[VAL_19]]{{\[}}%[[VAL_146]]] : memref<11xindex> -// CHECK: %[[VAL_148:.*]] = arith.addi %[[VAL_143]], %[[VAL_8]] : index -// CHECK: %[[VAL_149:.*]] = memref.load %[[VAL_19]]{{\[}}%[[VAL_148]]] : memref<11xindex> -// CHECK: %[[VAL_150:.*]] = arith.cmpi ult, %[[VAL_147]], %[[VAL_149]] : index -// CHECK: %[[VAL_151:.*]] = scf.if %[[VAL_150]] -> (index) { -// CHECK: %[[VAL_152:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_147]]] : memref -// CHECK: %[[VAL_153:.*]] = arith.cmpi eq, %[[VAL_152]], %[[VAL_88]] : index -// CHECK: %[[VAL_154:.*]] = scf.if %[[VAL_153]] -> (index) { -// CHECK: %[[VAL_155:.*]] = arith.addi %[[VAL_147]], %[[VAL_7]] : index -// CHECK: memref.store %[[VAL_155]], %[[VAL_19]]{{\[}}%[[VAL_146]]] : memref<11xindex> -// CHECK: scf.yield %[[VAL_155]] : index +// CHECK: %[[VAL_121:.*]]:2 = scf.for %[[VAL_122:.*]] = %[[VAL_8]] to %[[VAL_65]]#3 step %[[VAL_7]] iter_args(%[[VAL_123:.*]] = %[[VAL_5]], %[[VAL_124:.*]] = %[[VAL_10]]) -> (index, i1) { +// CHECK: %[[VAL_125:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_122]]] : memref<9xindex> +// CHECK: %[[VAL_126:.*]] = arith.addi %[[VAL_122]], %[[VAL_6]] : index +// CHECK: %[[VAL_127:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_126]]] : memref<9xindex> +// CHECK: %[[VAL_128:.*]] = arith.cmpi ult, %[[VAL_125]], %[[VAL_127]] : index +// CHECK: %[[VAL_129:.*]] = scf.if %[[VAL_128]] -> (index) { +// CHECK: %[[VAL_130:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_125]]] : memref +// CHECK: %[[VAL_131:.*]] = arith.cmpi eq, %[[VAL_130]], %[[VAL_74]] : index +// CHECK: %[[VAL_132:.*]] = scf.if %[[VAL_131]] -> (index) { +// CHECK: %[[VAL_133:.*]] = arith.addi %[[VAL_125]], %[[VAL_7]] : index +// CHECK: memref.store %[[VAL_133]], %[[VAL_17]]{{\[}}%[[VAL_122]]] : memref<9xindex> +// CHECK: scf.yield %[[VAL_133]] : index // CHECK: } else { -// CHECK: scf.yield %[[VAL_147]] : index +// CHECK: scf.yield %[[VAL_125]] : index // CHECK: } -// CHECK: scf.yield %[[VAL_154]] : index +// CHECK: scf.yield %[[VAL_132]] : index // CHECK: } else { -// CHECK: scf.yield %[[VAL_147]] : index +// CHECK: scf.yield %[[VAL_125]] : index // CHECK: } -// CHECK: %[[VAL_156:.*]] = arith.cmpi ult, %[[VAL_151]], %[[VAL_149]] : index -// CHECK: %[[VAL_157:.*]] = scf.if %[[VAL_156]] -> (index) { -// CHECK: %[[VAL_158:.*]] = memref.load %[[VAL_17]]{{\[}}%[[VAL_151]]] : memref -// CHECK: scf.yield %[[VAL_158]] : index +// CHECK: %[[VAL_134:.*]] = arith.cmpi ult, %[[VAL_129]], %[[VAL_127]] : index +// CHECK: %[[VAL_135:.*]] = scf.if %[[VAL_134]] -> (index) { +// CHECK: %[[VAL_136:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_129]]] : memref +// CHECK: scf.yield %[[VAL_136]] : index // CHECK: } else { -// CHECK: scf.yield %[[VAL_144]] : index +// CHECK: scf.yield %[[VAL_123]] : index // CHECK: } -// CHECK: %[[VAL_159:.*]] = arith.ori %[[VAL_156]], %[[VAL_145]] : i1 -// CHECK: %[[VAL_160:.*]] = arith.cmpi ult, %[[VAL_157]], %[[VAL_144]] : index -// CHECK: %[[VAL_161:.*]] = arith.select %[[VAL_160]], %[[VAL_157]], %[[VAL_144]] : index -// CHECK: scf.yield %[[VAL_161]], %[[VAL_159]] : index, i1 +// CHECK: %[[VAL_137:.*]] = arith.ori %[[VAL_134]], %[[VAL_124]] : i1 +// CHECK: %[[VAL_138:.*]] = arith.cmpi ult, %[[VAL_135]], %[[VAL_123]] : index +// CHECK: %[[VAL_139:.*]] = arith.select %[[VAL_138]], %[[VAL_135]], %[[VAL_123]] : index +// CHECK: scf.yield %[[VAL_139]], %[[VAL_137]] : index, i1 // CHECK: } -// CHECK: %[[VAL_162:.*]] = arith.addi %[[VAL_163:.*]]#0, %[[VAL_7]] : index -// CHECK: %[[VAL_164:.*]] = arith.addi %[[VAL_163]]#0, %[[VAL_3]] : index -// CHECK: %[[VAL_165:.*]] = arith.cmpi uge, %[[VAL_162]], %[[VAL_6]] : index -// CHECK: %[[VAL_166:.*]] = arith.select %[[VAL_165]], %[[VAL_164]], %[[VAL_10]] : index -// CHECK: scf.yield %[[VAL_163]]#0, %[[VAL_163]]#1, %[[VAL_166]] : index, i1, index +// CHECK: %[[VAL_140:.*]] = arith.addi %[[VAL_141:.*]]#0, %[[VAL_7]] : index +// CHECK: %[[VAL_142:.*]] = arith.addi %[[VAL_141]]#0, %[[VAL_3]] : index +// CHECK: %[[VAL_143:.*]] = arith.cmpi uge, %[[VAL_140]], %[[VAL_6]] : index +// CHECK: %[[VAL_144:.*]] = arith.select %[[VAL_143]], %[[VAL_142]], %[[VAL_8]] : index +// CHECK: scf.yield %[[VAL_141]]#0, %[[VAL_141]]#1, %[[VAL_144]] : index, i1, index // CHECK: } -// CHECK: %[[VAL_167:.*]] = arith.addi %[[VAL_89]], %[[VAL_7]] : index -// CHECK: %[[VAL_168:.*]] = arith.cmpi ugt, %[[VAL_169:.*]]#2, %[[VAL_167]] : index -// CHECK: %[[VAL_170:.*]] = arith.select %[[VAL_168]], %[[VAL_169]]#2, %[[VAL_167]] : index -// CHECK: %[[VAL_171:.*]] = arith.addi %[[VAL_170]], %[[VAL_6]] : index -// CHECK: %[[VAL_172:.*]] = arith.cmpi ule, %[[VAL_171]], %[[VAL_5]] : index -// CHECK: %[[VAL_173:.*]] = arith.andi %[[VAL_169]]#1, %[[VAL_172]] : i1 -// CHECK: scf.yield %[[VAL_173]], %[[VAL_169]]#0, %[[VAL_170]], %[[VAL_135]] : i1, index, index, tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_145:.*]] = arith.addi %[[VAL_75]], %[[VAL_7]] : index +// CHECK: %[[VAL_146:.*]] = arith.cmpi ugt, %[[VAL_147:.*]]#2, %[[VAL_145]] : index +// CHECK: %[[VAL_148:.*]] = arith.select %[[VAL_146]], %[[VAL_147]]#2, %[[VAL_145]] : index +// CHECK: %[[VAL_149:.*]] = arith.addi %[[VAL_148]], %[[VAL_6]] : index +// CHECK: %[[VAL_150:.*]] = arith.cmpi ule, %[[VAL_149]], %[[VAL_5]] : index +// CHECK: %[[VAL_151:.*]] = arith.andi %[[VAL_147]]#1, %[[VAL_150]] : i1 +// CHECK: scf.yield %[[VAL_151]], %[[VAL_147]]#0, %[[VAL_148]], %[[VAL_115]] : i1, index, index, tensor<6x6xi32, #sparse> // CHECK: } -// CHECK: memref.store %[[VAL_10]], %[[VAL_20]]{{\[}}%[[VAL_7]]] : memref<5xindex> -// CHECK: %[[VAL_174:.*]] = arith.cmpi ugt, %[[VAL_33]], %[[VAL_34]] : index -// CHECK: %[[VAL_175:.*]]:3 = scf.if %[[VAL_174]] -> (index, i1, index) { -// CHECK: %[[VAL_176:.*]] = arith.addi %[[VAL_34]], %[[VAL_7]] : index -// CHECK: scf.yield %[[VAL_33]], %[[VAL_2]], %[[VAL_176]] : index, i1, index +// CHECK: %[[VAL_152:.*]] = arith.cmpi ugt, %[[VAL_31]], %[[VAL_32]] : index +// CHECK: %[[VAL_153:.*]]:3 = scf.if %[[VAL_152]] -> (index, i1, index) { +// CHECK: %[[VAL_154:.*]] = arith.addi %[[VAL_32]], %[[VAL_7]] : index +// CHECK: scf.yield %[[VAL_31]], %[[VAL_2]], %[[VAL_154]] : index, i1, index // CHECK: } else { -// CHECK: %[[VAL_177:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_10]]] : memref<5xindex> -// CHECK: %[[VAL_178:.*]]:2 = scf.for %[[VAL_179:.*]] = %[[VAL_10]] to %[[VAL_177]] step %[[VAL_7]] iter_args(%[[VAL_180:.*]] = %[[VAL_5]], %[[VAL_181:.*]] = %[[VAL_12]]) -> (index, i1) { -// CHECK: %[[VAL_182:.*]] = arith.addi %[[VAL_179]], %[[VAL_9]] : index -// CHECK: %[[VAL_183:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_182]]] : memref<5xindex> -// CHECK: %[[VAL_184:.*]] = arith.addi %[[VAL_179]], %[[VAL_6]] : index -// CHECK: %[[VAL_185:.*]] = memref.load %[[VAL_20]]{{\[}}%[[VAL_184]]] : memref<5xindex> -// CHECK: %[[VAL_186:.*]] = arith.cmpi ult, %[[VAL_183]], %[[VAL_185]] : index -// CHECK: %[[VAL_187:.*]] = scf.if %[[VAL_186]] -> (index) { -// CHECK: %[[VAL_188:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_183]]] : memref -// CHECK: %[[VAL_189:.*]] = arith.cmpi eq, %[[VAL_188]], %[[VAL_33]] : index -// CHECK: %[[VAL_190:.*]] = scf.if %[[VAL_189]] -> (index) { -// CHECK: %[[VAL_191:.*]] = arith.addi %[[VAL_183]], %[[VAL_7]] : index -// CHECK: memref.store %[[VAL_191]], %[[VAL_20]]{{\[}}%[[VAL_182]]] : memref<5xindex> -// CHECK: scf.yield %[[VAL_191]] : index -// CHECK: } else { -// CHECK: scf.yield %[[VAL_183]] : index -// CHECK: } -// CHECK: scf.yield %[[VAL_190]] : index -// CHECK: } else { -// CHECK: scf.yield %[[VAL_183]] : index -// CHECK: } -// CHECK: %[[VAL_192:.*]] = arith.cmpi ult, %[[VAL_187]], %[[VAL_185]] : index -// CHECK: %[[VAL_193:.*]] = scf.if %[[VAL_192]] -> (index) { -// CHECK: %[[VAL_194:.*]] = memref.load %[[VAL_15]]{{\[}}%[[VAL_187]]] : memref -// CHECK: scf.yield %[[VAL_194]] : index +// CHECK: %[[VAL_155:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_8]]] : memref<3xindex> +// CHECK: %[[VAL_156:.*]] = memref.load %[[VAL_18]]{{\[}}%[[VAL_7]]] : memref<3xindex> +// CHECK: %[[VAL_157:.*]] = arith.cmpi ult, %[[VAL_155]], %[[VAL_156]] : index +// CHECK: %[[VAL_158:.*]] = scf.if %[[VAL_157]] -> (index) { +// CHECK: %[[VAL_159:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_155]]] : memref +// CHECK: %[[VAL_160:.*]] = arith.cmpi eq, %[[VAL_159]], %[[VAL_31]] : index +// CHECK: %[[VAL_161:.*]] = scf.if %[[VAL_160]] -> (index) { +// CHECK: %[[VAL_162:.*]] = arith.addi %[[VAL_155]], %[[VAL_7]] : index +// CHECK: memref.store %[[VAL_162]], %[[VAL_18]]{{\[}}%[[VAL_8]]] : memref<3xindex> +// CHECK: scf.yield %[[VAL_162]] : index // CHECK: } else { -// CHECK: scf.yield %[[VAL_180]] : index +// CHECK: scf.yield %[[VAL_155]] : index // CHECK: } -// CHECK: %[[VAL_195:.*]] = arith.ori %[[VAL_192]], %[[VAL_181]] : i1 -// CHECK: %[[VAL_196:.*]] = arith.cmpi ult, %[[VAL_193]], %[[VAL_180]] : index -// CHECK: %[[VAL_197:.*]] = arith.select %[[VAL_196]], %[[VAL_193]], %[[VAL_180]] : index -// CHECK: scf.yield %[[VAL_197]], %[[VAL_195]] : index, i1 +// CHECK: scf.yield %[[VAL_161]] : index +// CHECK: } else { +// CHECK: scf.yield %[[VAL_155]] : index +// CHECK: } +// CHECK: %[[VAL_163:.*]] = arith.cmpi ult, %[[VAL_158]], %[[VAL_156]] : index +// CHECK: %[[VAL_164:.*]] = scf.if %[[VAL_163]] -> (index) { +// CHECK: %[[VAL_165:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_158]]] : memref +// CHECK: scf.yield %[[VAL_165]] : index +// CHECK: } else { +// CHECK: scf.yield %[[VAL_5]] : index // CHECK: } -// CHECK: %[[VAL_198:.*]] = arith.addi %[[VAL_199:.*]]#0, %[[VAL_7]] : index -// CHECK: %[[VAL_200:.*]] = arith.addi %[[VAL_199]]#0, %[[VAL_3]] : index -// CHECK: %[[VAL_201:.*]] = arith.cmpi uge, %[[VAL_198]], %[[VAL_6]] : index -// CHECK: %[[VAL_202:.*]] = arith.select %[[VAL_201]], %[[VAL_200]], %[[VAL_10]] : index -// CHECK: scf.yield %[[VAL_199]]#0, %[[VAL_199]]#1, %[[VAL_202]] : index, i1, index +// CHECK: %[[VAL_166:.*]] = arith.cmpi ult, %[[VAL_164]], %[[VAL_5]] : index +// CHECK: %[[VAL_167:.*]] = arith.select %[[VAL_166]], %[[VAL_164]], %[[VAL_5]] : index +// CHECK: %[[VAL_168:.*]] = arith.addi %[[VAL_167]], %[[VAL_7]] : index +// CHECK: %[[VAL_169:.*]] = arith.addi %[[VAL_167]], %[[VAL_3]] : index +// CHECK: %[[VAL_170:.*]] = arith.cmpi uge, %[[VAL_168]], %[[VAL_6]] : index +// CHECK: %[[VAL_171:.*]] = arith.select %[[VAL_170]], %[[VAL_169]], %[[VAL_8]] : index +// CHECK: scf.yield %[[VAL_167]], %[[VAL_163]], %[[VAL_171]] : index, i1, index // CHECK: } -// CHECK: %[[VAL_203:.*]] = arith.addi %[[VAL_34]], %[[VAL_7]] : index -// CHECK: %[[VAL_204:.*]] = arith.cmpi ugt, %[[VAL_205:.*]]#2, %[[VAL_203]] : index -// CHECK: %[[VAL_206:.*]] = arith.select %[[VAL_204]], %[[VAL_205]]#2, %[[VAL_203]] : index -// CHECK: %[[VAL_207:.*]] = arith.addi %[[VAL_206]], %[[VAL_6]] : index -// CHECK: %[[VAL_208:.*]] = arith.cmpi ule, %[[VAL_207]], %[[VAL_5]] : index -// CHECK: %[[VAL_209:.*]] = arith.andi %[[VAL_205]]#1, %[[VAL_208]] : i1 -// CHECK: scf.yield %[[VAL_209]], %[[VAL_205]]#0, %[[VAL_206]], %[[VAL_210:.*]]#2 : i1, index, index, tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_172:.*]] = arith.addi %[[VAL_32]], %[[VAL_7]] : index +// CHECK: %[[VAL_173:.*]] = arith.cmpi ugt, %[[VAL_174:.*]]#2, %[[VAL_172]] : index +// CHECK: %[[VAL_175:.*]] = arith.select %[[VAL_173]], %[[VAL_174]]#2, %[[VAL_172]] : index +// CHECK: %[[VAL_176:.*]] = arith.addi %[[VAL_175]], %[[VAL_6]] : index +// CHECK: %[[VAL_177:.*]] = arith.cmpi ule, %[[VAL_176]], %[[VAL_5]] : index +// CHECK: %[[VAL_178:.*]] = arith.andi %[[VAL_174]]#1, %[[VAL_177]] : i1 +// CHECK: scf.yield %[[VAL_178]], %[[VAL_174]]#0, %[[VAL_175]], %[[VAL_179:.*]]#2 : i1, index, index, tensor<6x6xi32, #sparse> // CHECK: } -// CHECK: %[[VAL_211:.*]] = sparse_tensor.load %[[VAL_212:.*]]#2 hasInserts : tensor<6x6xi32, #sparse{{[0-9]*}}> -// CHECK: return %[[VAL_211]] : tensor<6x6xi32, #sparse{{[0-9]*}}> +// CHECK: %[[VAL_180:.*]] = sparse_tensor.load %[[VAL_181:.*]]#2 hasInserts : tensor<6x6xi32, #sparse> +// CHECK: return %[[VAL_180]] : tensor<6x6xi32, #sparse> // CHECK: } func.func @conv2d_all_sparse_CSR(%arg0: tensor<8x8xi32, #DCSR>, %arg1: tensor<3x3xi32>) -> tensor<6x6xi32, #DCSR> { diff --git a/mlir/test/Dialect/SparseTensor/spy_sddmm_bsr.mlir b/mlir/test/Dialect/SparseTensor/spy_sddmm_bsr.mlir new file mode 100755 index 000000000000..ed8d63987896 --- /dev/null +++ b/mlir/test/Dialect/SparseTensor/spy_sddmm_bsr.mlir @@ -0,0 +1,103 @@ +// RUN: mlir-opt %s --sparse-reinterpret-map -sparsification | FileCheck %s + +// +// A SDDMM implementation with "spy" function and +// in-place update of the sampling sparse matrix. +// + +#BSR = #sparse_tensor.encoding<{ + map = (i, j) -> ( + i floordiv 2 : dense, + j floordiv 2 : compressed, + i mod 2 : dense, + j mod 2 : dense) +}> + +#trait_SDDMM = { + indexing_maps = [ + affine_map<(i,j,k) -> (i,k)>, // A + affine_map<(i,j,k) -> (k,j)>, // B + affine_map<(i,j,k) -> (i,j)> // S (in/out) + ], + iterator_types = ["parallel", "parallel", "reduction"], + doc = "S(i,j) += spy[S(i,j)] x SUM_k A(i,k) B(k,j)" +} + +// +// CHECK: #[[$BSR:.+]] = #sparse_tensor.encoding<{ map = (d0, d1) -> (d0 floordiv 2 : dense, d1 floordiv 2 : compressed, d0 mod 2 : dense, d1 mod 2 : dense) }> +// CHECK: #[[$MAP:.+]] = #sparse_tensor.encoding<{ map = (d0, d1, d2, d3) -> (d0 : dense, d1 : compressed, d2 : dense, d3 : dense) }> +// +// CHECK-LABEL: func.func @SDDMM_block( +// CHECK-SAME: %[[VAL_0:.*]]: tensor, +// CHECK-SAME: %[[VAL_1:.*]]: tensor, +// CHECK-SAME: %[[VAL_2:.*]]: tensor) -> tensor { +// CHECK-DAG: %[[VAL_3:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[VAL_4:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[VAL_5:.*]] = arith.constant 2 : index +// CHECK-DAG: %[[VAL_6:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[VAL_7:.*]] = sparse_tensor.reinterpret_map %[[VAL_0]] : tensor to tensor +// CHECK: %[[VAL_8:.*]] = tensor.dim %[[VAL_1]], %[[VAL_3]] : tensor +// CHECK: %[[VAL_9:.*]] = bufferization.to_memref %[[VAL_1]] : memref +// CHECK: %[[VAL_10:.*]] = bufferization.to_memref %[[VAL_2]] : memref +// CHECK: %[[VAL_11:.*]] = sparse_tensor.lvl %[[VAL_7]], %[[VAL_4]] : tensor +// CHECK: %[[VAL_12:.*]] = sparse_tensor.positions %[[VAL_7]] {level = 1 : index} : tensor to memref +// CHECK: %[[VAL_13:.*]] = sparse_tensor.coordinates %[[VAL_7]] {level = 1 : index} : tensor to memref +// CHECK: %[[VAL_14:.*]] = sparse_tensor.values %[[VAL_7]] : tensor to memref +// CHECK: scf.for %[[VAL_15:.*]] = %[[VAL_4]] to %[[VAL_11]] step %[[VAL_3]] { +// CHECK: %[[VAL_16:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_15]]] : memref +// CHECK: %[[VAL_17:.*]] = arith.addi %[[VAL_15]], %[[VAL_3]] : index +// CHECK: %[[VAL_18:.*]] = memref.load %[[VAL_12]]{{\[}}%[[VAL_17]]] : memref +// CHECK: scf.for %[[VAL_19:.*]] = %[[VAL_16]] to %[[VAL_18]] step %[[VAL_3]] { +// CHECK: %[[VAL_20:.*]] = memref.load %[[VAL_13]]{{\[}}%[[VAL_19]]] : memref +// CHECK: scf.for %[[VAL_21:.*]] = %[[VAL_4]] to %[[VAL_5]] step %[[VAL_3]] { +// CHECK: %[[VAL_22:.*]] = arith.muli %[[VAL_19]], %[[VAL_5]] : index +// CHECK: %[[VAL_23:.*]] = arith.addi %[[VAL_22]], %[[VAL_21]] : index +// CHECK: scf.for %[[VAL_24:.*]] = %[[VAL_4]] to %[[VAL_5]] step %[[VAL_3]] { +// CHECK: %[[VAL_25:.*]] = arith.muli %[[VAL_23]], %[[VAL_5]] : index +// CHECK: %[[VAL_26:.*]] = arith.addi %[[VAL_25]], %[[VAL_24]] : index +// CHECK: %[[VAL_27:.*]] = scf.for %[[VAL_28:.*]] = %[[VAL_4]] to %[[VAL_8]] step %[[VAL_3]] iter_args(%[[VAL_29:.*]] = %[[VAL_6]]) -> (f32) { +// CHECK: %[[VAL_30:.*]] = arith.muli %[[VAL_15]], %[[VAL_5]] : index +// CHECK: %[[VAL_31:.*]] = arith.addi %[[VAL_30]], %[[VAL_21]] : index +// CHECK: %[[VAL_32:.*]] = memref.load %[[VAL_9]]{{\[}}%[[VAL_31]], %[[VAL_28]]] : memref +// CHECK: %[[VAL_33:.*]] = arith.muli %[[VAL_20]], %[[VAL_5]] : index +// CHECK: %[[VAL_34:.*]] = arith.addi %[[VAL_33]], %[[VAL_24]] : index +// CHECK: %[[VAL_35:.*]] = memref.load %[[VAL_10]]{{\[}}%[[VAL_28]], %[[VAL_34]]] : memref +// CHECK: %[[VAL_36:.*]] = arith.mulf %[[VAL_32]], %[[VAL_35]] : f32 +// CHECK: %[[VAL_37:.*]] = arith.addf %[[VAL_29]], %[[VAL_36]] : f32 +// CHECK: scf.yield %[[VAL_37]] : f32 +// CHECK: } {"Emitted from" = "linalg.generic"} +// CHECK: memref.store %[[VAL_27]], %[[VAL_14]]{{\[}}%[[VAL_26]]] : memref +// CHECK: } {"Emitted from" = "linalg.generic"} +// CHECK: } {"Emitted from" = "linalg.generic"} +// CHECK: } {"Emitted from" = "linalg.generic"} +// CHECK: } {"Emitted from" = "linalg.generic"} +// CHECK: %[[VAL_38:.*]] = sparse_tensor.load %[[VAL_7]] : tensor +// CHECK: %[[VAL_39:.*]] = sparse_tensor.reinterpret_map %[[VAL_38]] : tensor to tensor +// CHECK: return %[[VAL_39]] : tensor +// CHECK: } +module { + func.func @SDDMM_block(%args: tensor, + %arga: tensor, + %argb: tensor) -> tensor { + %result = linalg.generic #trait_SDDMM + ins(%arga, %argb: tensor, tensor) + outs(%args: tensor) { + ^bb(%a: f32, %b: f32, %s: f32): + %f0 = arith.constant 0.0 : f32 + %u = sparse_tensor.unary %s : f32 to f32 + present={ + ^bb0(%p: f32): + %mul = arith.mulf %a, %b : f32 + sparse_tensor.yield %mul : f32 + } + absent={} + %r = sparse_tensor.reduce %s, %u, %f0 : f32 { + ^bb0(%p: f32, %q: f32): + %add = arith.addf %p, %q : f32 + sparse_tensor.yield %add : f32 + } + linalg.yield %r : f32 + } -> tensor + return %result : tensor + } +} diff --git a/mlir/test/Dialect/Tensor/canonicalize.mlir b/mlir/test/Dialect/Tensor/canonicalize.mlir index 84c44a09aa3d..7d7d221c1e8e 100644 --- a/mlir/test/Dialect/Tensor/canonicalize.mlir +++ b/mlir/test/Dialect/Tensor/canonicalize.mlir @@ -2018,3 +2018,20 @@ func.func @invalid_slice_ops(%t: tensor, %t2: tensor) -> tensor into tensor return %1 : tensor } + +// ----- + +// CHECK-LABEL: func @generate_negative_size_verifies( +// CHECK: %[[c:.*]] = arith.constant -8 : index +// CHECK: tensor.generate %[[c]] +// CHECK: : tensor +func.func @generate_negative_size_verifies() -> tensor { + %cst = arith.constant 0 : i32 + %c0 = arith.constant 0 : index + %size = affine.max affine_map<(d0) -> (d0 mod 64 - 8)>(%c0) + %tensor = tensor.generate %size { + ^bb0(%arg0: index, %arg1: index): + tensor.yield %cst : i32 + } : tensor + return %tensor : tensor +} diff --git a/mlir/test/Dialect/Tensor/invalid-canonicalize.mlir b/mlir/test/Dialect/Tensor/invalid-canonicalize.mlir deleted file mode 100644 index decfd55eacc9..000000000000 --- a/mlir/test/Dialect/Tensor/invalid-canonicalize.mlir +++ /dev/null @@ -1,15 +0,0 @@ -// RUN: mlir-opt <%s -split-input-file -verify-diagnostics -canonicalize - -// ----- - -func.func @indirectly_generate_negative_size() -> tensor { - %cst = arith.constant 0 : i32 - %c0 = arith.constant 0 : index - %size = affine.max affine_map<(d0) -> (d0 mod 64 - 8)>(%c0) - // expected-error@+1 {{tensor dimensions must be non-negative}} - %tensor = tensor.generate %size { - ^bb0(%arg0: index, %arg1: index): - tensor.yield %cst : i32 - } : tensor - return %tensor : tensor -} diff --git a/mlir/test/Dialect/Tensor/invalid.mlir b/mlir/test/Dialect/Tensor/invalid.mlir index 9b6c2327879c..bdada43e325c 100644 --- a/mlir/test/Dialect/Tensor/invalid.mlir +++ b/mlir/test/Dialect/Tensor/invalid.mlir @@ -163,19 +163,6 @@ func.func @tensor.generate(%m : index, %n : index) // ----- -func.func @generate_negative_size() -> tensor { - %cst = arith.constant 0 : i32 - %size = index.constant -128 - // expected-error@+1 {{tensor dimensions must be non-negative}} - %tensor = tensor.generate %size { - ^bb0(%arg0: index, %arg1: index): - tensor.yield %cst : i32 - } : tensor - return %tensor : tensor -} - -// ----- - func.func @tensor.reshape_element_type_mismatch( %buf: tensor<*xf32>, %shape: tensor<1xi32>) { // expected-error @+1 {{element types of source and destination tensor types should be the same}} diff --git a/mlir/test/Dialect/Vector/invalid.mlir b/mlir/test/Dialect/Vector/invalid.mlir index edb2689364a9..ad248d1e14e7 100644 --- a/mlir/test/Dialect/Vector/invalid.mlir +++ b/mlir/test/Dialect/Vector/invalid.mlir @@ -332,6 +332,28 @@ func.func @test_vector.transfer_read(%arg0: memref) { // ----- +#map1 = affine_map<(d0, d1, d2) -> (d0, 0, 0)> +func.func @main(%m: memref<1xi32>, %2: vector<1x32xi1>) -> vector<1x32xi32> { + %0 = arith.constant 1 : index + %1 = arith.constant 1 : i32 + // expected-error@+1 {{expected the same rank for the vector and the results of the permutation map}} + %3 = vector.transfer_read %m[%0], %1, %2 { permutation_map = #map1 } : memref<1xi32>, vector<1x32xi32> + return %3 : vector<1x32xi32> +} + +// ----- + +#map1 = affine_map<(d0, d1, d2) -> (d0, 0, 0)> +func.func @test_vector.transfer_write(%m: memref<1xi32>, %2: vector<1x32xi32>) -> vector<1x32xi32> { + %0 = arith.constant 1 : index + %1 = arith.constant 1 : i32 + // expected-error@+1 {{expected the same rank for the vector and the results of the permutation map}} + %3 = vector.transfer_write %2, %m[%0], %1 { permutation_map = #map1 } : vector<1x32xi32>, memref<1xi32> + return %3 : vector<1x32xi32> +} + +// ----- + func.func @test_vector.transfer_read(%arg0: vector<4x3xf32>) { %c3 = arith.constant 3 : index %f0 = arith.constant 0.0 : f32 diff --git a/mlir/test/IR/custom-print-parse.mlir b/mlir/test/IR/custom-print-parse.mlir new file mode 100644 index 000000000000..b157fd1b1ea3 --- /dev/null +++ b/mlir/test/IR/custom-print-parse.mlir @@ -0,0 +1,17 @@ +// RUN: mlir-opt %s | FileCheck %s + +// CHECK-LABEL: module @dimension_list +module @dimension_list { + // CHECK: test.custom_dimension_list_attr dimension_list = [] + test.custom_dimension_list_attr dimension_list = [] + // CHECK: test.custom_dimension_list_attr dimension_list = 3 + test.custom_dimension_list_attr dimension_list = 3 + // CHECK: test.custom_dimension_list_attr dimension_list = 0 + test.custom_dimension_list_attr dimension_list = 0 + // CHECK: test.custom_dimension_list_attr dimension_list = 1x2 + test.custom_dimension_list_attr dimension_list = 1x2 + // CHECK: test.custom_dimension_list_attr dimension_list = ? + test.custom_dimension_list_attr dimension_list = ? + // CHECK: test.custom_dimension_list_attr dimension_list = ?x? + test.custom_dimension_list_attr dimension_list = ?x? +} diff --git a/mlir/test/IR/invalid-custom-print-parse.mlir b/mlir/test/IR/invalid-custom-print-parse.mlir new file mode 100644 index 000000000000..456b16c91bc0 --- /dev/null +++ b/mlir/test/IR/invalid-custom-print-parse.mlir @@ -0,0 +1,16 @@ +// RUN: mlir-opt %s -split-input-file -verify-diagnostics + +// expected-error@+2 {{invalid dimension}} +// expected-error@+1 {{custom op 'test.custom_dimension_list_attr' Failed parsing dimension list.}} +test.custom_dimension_list_attr dimension_list = 1x-1 + +// ----- + +// expected-error@+1 {{custom op 'test.custom_dimension_list_attr' Failed parsing dimension list. Did you mean an empty list? It must be denoted by "[]".}} +test.custom_dimension_list_attr dimension_list = -1 + +// ----- + +// expected-error@+2 {{expected ']'}} +// expected-error@+1 {{custom op 'test.custom_dimension_list_attr' Failed parsing dimension list.}} +test.custom_dimension_list_attr dimension_list = [2x3] diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir index 7825e8fe9baf..6c35e2b51ed8 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/dual_sparse_conv_2d.mlir @@ -41,8 +41,8 @@ module { func.func @conv2d(%input: tensor<8x8xi32>, - %filter: tensor<3x3xi32>, - %output: tensor<6x6xi32>) -> tensor<6x6xi32> { + %filter: tensor<3x3xi32>, + %output: tensor<6x6xi32>) -> tensor<6x6xi32> { %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32>, tensor<3x3xi32>) outs (%output: tensor<6x6xi32>) -> tensor<6x6xi32> @@ -50,7 +50,7 @@ module { } func.func @conv2d_all_sparse_DCSR(%input: tensor<8x8xi32, #DCSR>, - %filter: tensor<3x3xi32, #DCSR>) -> tensor<6x6xi32, #DCSR> { + %filter: tensor<3x3xi32, #DCSR>) -> tensor<6x6xi32, #DCSR> { %s = tensor.empty() : tensor<6x6xi32, #DCSR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #DCSR>, tensor<3x3xi32, #DCSR>) @@ -59,7 +59,7 @@ module { } func.func @conv2d_all_sparse_CSR(%input: tensor<8x8xi32, #CSR>, - %filter: tensor<3x3xi32, #CSR>) -> tensor<6x6xi32, #CSR> { + %filter: tensor<3x3xi32, #CSR>) -> tensor<6x6xi32, #CSR> { %s = tensor.empty() : tensor<6x6xi32, #CSR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #CSR>, tensor<3x3xi32, #CSR>) @@ -68,7 +68,7 @@ module { } func.func @conv2d_all_sparse_CD(%input: tensor<8x8xi32, #CDR>, - %filter: tensor<3x3xi32, #CDR>) -> tensor<6x6xi32, #CDR> { + %filter: tensor<3x3xi32, #CDR>) -> tensor<6x6xi32, #CDR> { %s = tensor.empty() : tensor<6x6xi32, #CDR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #CDR>, tensor<3x3xi32, #CDR>) @@ -77,7 +77,7 @@ module { } func.func @conv2d_all_sparse_CSC(%input: tensor<8x8xi32, #CSC>, - %filter: tensor<3x3xi32, #CSC>) -> tensor<6x6xi32, #CSC> { + %filter: tensor<3x3xi32, #CSC>) -> tensor<6x6xi32, #CSC> { %s = tensor.empty() : tensor<6x6xi32, #CSC> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #CSC>, tensor<3x3xi32, #CSC>) diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir index c9abaa2aff63..9295ee8b469c 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_1d_nwc_wcf.mlir @@ -25,7 +25,6 @@ // // Do the same run, but now with direct IR generation and vectorization. // REDEFINE: %{sparsifier_opts} = enable-runtime-library=false enable-buffer-initialization=true vl=2 reassociate-fp-reductions=true enable-index-optimizations=true - // RUN: %{compile} | %{run} | FileCheck %s // // Do the same run, but now with direct IR generation and VLA vectorization. diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir index 80946f538852..41071ea700fb 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d.mlir @@ -25,7 +25,6 @@ // // Do the same run, but now with direct IR generation and vectorization. // REDEFINE: %{sparsifier_opts} = enable-runtime-library=false vl=2 reassociate-fp-reductions=true enable-index-optimizations=true - // RUN: %{compile} | %{run} | FileCheck %s // // Do the same run, but now with direct IR generation and VLA vectorization. @@ -46,8 +45,8 @@ module { func.func @conv2d(%input: tensor<8x8xi32>, - %filter: tensor<3x3xi32>, - %output: tensor<6x6xi32>) -> tensor<6x6xi32> { + %filter: tensor<3x3xi32>, + %output: tensor<6x6xi32>) -> tensor<6x6xi32> { %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32>, tensor<3x3xi32>) outs (%output: tensor<6x6xi32>) -> tensor<6x6xi32> @@ -56,21 +55,21 @@ module { func.func @conv2d_CSR_dense_rotated(%arg0: tensor<8x8xi32, #CSR>, %arg1: tensor<3x3xi32>) -> tensor<6x6xi32> { - %s = tensor.empty() : tensor<6x6xi32> + %s = arith.constant dense<0> : tensor<6x6xi32> %0 = linalg.generic {indexing_maps = [#map, #map1, #map2], - iterator_types = ["parallel", "reduction", "reduction", "parallel"]} - ins(%arg0, %arg1 : tensor<8x8xi32, #CSR>, tensor<3x3xi32>) - outs(%s : tensor<6x6xi32>) attrs = {sorted = true} { - ^bb0(%in: i32, %in_0: i32, %out: i32): - %1 = arith.muli %in, %in_0 : i32 - %2 = arith.addi %out, %1 : i32 - linalg.yield %2 : i32 + iterator_types = ["parallel", "reduction", "reduction", "parallel"]} + ins(%arg0, %arg1 : tensor<8x8xi32, #CSR>, tensor<3x3xi32>) + outs(%s : tensor<6x6xi32>) attrs = {sorted = true} { + ^bb0(%in: i32, %in_0: i32, %out: i32): + %1 = arith.muli %in, %in_0 : i32 + %2 = arith.addi %out, %1 : i32 + linalg.yield %2 : i32 } -> tensor<6x6xi32> return %0 : tensor<6x6xi32> } func.func @conv2d_sparse_out(%input: tensor<8x8xi32>, - %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #DCSR> { + %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #DCSR> { %s = tensor.empty() : tensor<6x6xi32, #DCSR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32>, tensor<3x3xi32>) @@ -79,7 +78,7 @@ module { } func.func @conv2d_all_sparse_DCSR(%input: tensor<8x8xi32, #DCSR>, - %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #DCSR> { + %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #DCSR> { %s = tensor.empty() : tensor<6x6xi32, #DCSR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #DCSR>, tensor<3x3xi32>) @@ -88,7 +87,7 @@ module { } func.func @conv2d_all_sparse_CSR(%input: tensor<8x8xi32, #CSR>, - %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #CSR> { + %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #CSR> { %s = tensor.empty() : tensor<6x6xi32, #CSR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #CSR>, tensor<3x3xi32>) @@ -97,7 +96,7 @@ module { } func.func @conv2d_all_sparse_CD(%input: tensor<8x8xi32, #CDR>, - %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #CDR> { + %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #CDR> { %s = tensor.empty() : tensor<6x6xi32, #CDR> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #CDR>, tensor<3x3xi32>) @@ -106,7 +105,7 @@ module { } func.func @conv2d_all_sparse_CSC(%input: tensor<8x8xi32, #CSC>, - %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #CSC> { + %filter: tensor<3x3xi32>) -> tensor<6x6xi32, #CSC> { %s = tensor.empty() : tensor<6x6xi32, #CSC> %0 = linalg.conv_2d ins (%input, %filter: tensor<8x8xi32, #CSC>, tensor<3x3xi32>) @@ -125,7 +124,6 @@ module { [ -1, 0, 1 ] ]> : tensor<3x3xi32> - %input = arith.constant dense<[ [ 1, 2, 3, 4, 0, 6, 7, 8 ], [ 2, 2, 4, 4, 0, 0, 6, 8 ], @@ -270,7 +268,6 @@ module { : tensor<6x6xi32>, vector<6x6xi32> vector.print %v : vector<6x6xi32> - // Release the resources. bufferization.dealloc_tensor %sparse_input_DCSR : tensor<8x8xi32, #DCSR> bufferization.dealloc_tensor %sparse_input_CSR : tensor<8x8xi32, #CSR> diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir new file mode 100755 index 000000000000..9e7bbe71038b --- /dev/null +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_55.mlir @@ -0,0 +1,220 @@ +//-------------------------------------------------------------------------------------------------- +// WHEN CREATING A NEW TEST, PLEASE JUST COPY & PASTE WITHOUT EDITS. +// +// Set-up that's shared across all tests in this directory. In principle, this +// config could be moved to lit.local.cfg. However, there are downstream users that +// do not use these LIT config files. Hence why this is kept inline. +// +// DEFINE: %{sparsifier_opts} = enable-runtime-library=true +// DEFINE: %{sparsifier_opts_sve} = enable-arm-sve=true %{sparsifier_opts} +// DEFINE: %{compile} = mlir-opt %s --sparsifier="%{sparsifier_opts}" +// DEFINE: %{compile_sve} = mlir-opt %s --sparsifier="%{sparsifier_opts_sve}" +// DEFINE: %{run_libs} = -shared-libs=%mlir_c_runner_utils,%mlir_runner_utils +// DEFINE: %{run_opts} = -e entry -entry-point-result=void +// DEFINE: %{run} = mlir-cpu-runner %{run_opts} %{run_libs} +// DEFINE: %{run_sve} = %mcr_aarch64_cmd --march=aarch64 --mattr="+sve" %{run_opts} %{run_libs} +// +// DEFINE: %{env} = +//-------------------------------------------------------------------------------------------------- + +// RUN: %{compile} | %{run} | FileCheck %s +// +// Do the same run, but now with direct IR generation. +// REDEFINE: %{sparsifier_opts} = enable-runtime-library=false +// RUN: %{compile} | %{run} | FileCheck %s +// +// Do the same run, but now with direct IR generation and vectorization. +// REDEFINE: %{sparsifier_opts} = enable-runtime-library=false vl=2 reassociate-fp-reductions=true enable-index-optimizations=true +// +// Do the same run, but now with direct IR generation and VLA vectorization. +// RUN: %if mlir_arm_sve_tests %{ %{compile_sve} | %{run_sve} | FileCheck %s %} + +#CSR = #sparse_tensor.encoding<{ + map = (d0, d1) -> (d0 : dense, + d1 : compressed) +}> + +#DCSR = #sparse_tensor.encoding<{ + map = (d0, d1) -> (d0 : compressed, + d1 : compressed) +}> + +// An example of a 2D convolution with sparse data and filter. +module { + func.func @conv2d(%input: tensor<10x10xi32>, + %filter: tensor<5x5xi32>, + %output: tensor<6x6xi32>) -> tensor<6x6xi32> { + %0 = linalg.conv_2d + ins (%input, %filter: tensor<10x10xi32>, tensor<5x5xi32>) + outs (%output: tensor<6x6xi32>) -> tensor<6x6xi32> + return %0 : tensor<6x6xi32> + } + + func.func @conv2d_ss(%input: tensor<10x10xi32, #CSR>, + %filter: tensor<5x5xi32, #CSR>, + %output: tensor<6x6xi32>) -> tensor<6x6xi32> { + %0 = linalg.conv_2d + ins (%input, %filter: tensor<10x10xi32, #CSR>, tensor<5x5xi32, #CSR>) + outs (%output: tensor<6x6xi32>) -> tensor<6x6xi32> + return %0 : tensor<6x6xi32> + } + + func.func @conv2d_bs(%input: tensor<10x10xi32, #DCSR>, + %filter: tensor<5x5xi32, #CSR>, + %output: tensor<6x6xi32>) -> tensor<6x6xi32> { + %0 = linalg.conv_2d + ins (%input, %filter: tensor<10x10xi32, #DCSR>, tensor<5x5xi32, #CSR>) + outs (%output: tensor<6x6xi32>) -> tensor<6x6xi32> + return %0 : tensor<6x6xi32> + } + + func.func @entry() { + %c0 = arith.constant 0 : index + %i0 = arith.constant 0 : i32 + + // Dense filter and input to "stress" test sparsity. + + %filter = arith.constant dense<[ + [ -1, -2, -3, -4, -5 ], + [ -6, -7, -8, -9, -10 ], + [ -11, -12, -13, -14, -15 ], + [ -16, -17, -18, -19, -20 ], + [ -21, -22, -23, -24, -25 ] + ]> : tensor<5x5xi32> + + %input = arith.constant dense<[ + [ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 ], + [ 10, 11, 12, 13, 14, 15, 16, 17, 18, 19 ], + [ 20, 21, 22, 23, 24, 25, 26, 27, 28, 29 ], + [ 30, 31, 32, 33, 34, 35, 36, 37, 38, 39 ], + [ 40, 41, 42, 43, 44, 45, 46, 47, 48, 49 ], + [ 50, 51, 52, 53, 54, 55, 56, 57, 58, 59 ], + [ 60, 61, 62, 63, 64, 65, 66, 67, 68, 69 ], + [ 70, 71, 72, 73, 74, 75, 76, 77, 78, 79 ], + [ 80, 81, 82, 83, 84, 85, 86, 87, 88, 89 ], + [ 90, 91, 92, 93, 94, 95, 96, 97, 98, 99 ] + ]> : tensor<10x10xi32> + + // Sparse filter and input to test true sparsity. + + %sfilter = arith.constant dense<[ + [ 0, -1, 0, -2, 0 ], + [ 0, 0, 0, 0, 0 ], + [ 0, 0, 8, 0, 0 ], + [ -3, 0, 0, -4, 0 ], + [ 0, 0, -5, 0, -6 ] + ]> : tensor<5x5xi32> + + %sinput = arith.constant dense<[ + [ 0, 1, 2, 3, 0, 0, 0, 0, 0, 0 ], + [ 0, 4, 0, 0, 5, 0, 0, 0, 0, 0 ], + [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], + [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], + [ 0, 0, 0, 0, 0, 0, 6, 0, 0, 7 ], + [ 0, 0, 0, 0, 0, 0, 0, 8, 0, 0 ], + [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], + [ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 ], + [ 0, 9, 0, 0, 0, 0, 0, 0, 0, 0 ], + [ 0, 0, 0, 0, 10, 0, 0, 0, 0, 0 ] + ]> : tensor<10x10xi32> + + // Set up sparse tensors. + + %input_CSR = sparse_tensor.convert %input : tensor<10x10xi32> to tensor<10x10xi32, #CSR> + %input_DCSR = sparse_tensor.convert %input : tensor<10x10xi32> to tensor<10x10xi32, #DCSR> + %filter_CSR = sparse_tensor.convert %filter : tensor<5x5xi32> to tensor<5x5xi32, #CSR> + + %sinput_CSR = sparse_tensor.convert %sinput : tensor<10x10xi32> to tensor<10x10xi32, #CSR> + %sinput_DCSR = sparse_tensor.convert %sinput : tensor<10x10xi32> to tensor<10x10xi32, #DCSR> + %sfilter_CSR = sparse_tensor.convert %sfilter : tensor<5x5xi32> to tensor<5x5xi32, #CSR> + + // Call the kernels with stress input. + %output0 = arith.constant dense<0> : tensor<6x6xi32> + %0 = call @conv2d(%input, %filter, %output0) + : (tensor<10x10xi32>, tensor<5x5xi32>, tensor<6x6xi32>) -> tensor<6x6xi32> + %output1 = arith.constant dense<0> : tensor<6x6xi32> + %1 = call @conv2d_ss(%input_CSR, %filter_CSR, %output1) + : (tensor<10x10xi32, #CSR>, tensor<5x5xi32, #CSR>, tensor<6x6xi32>) -> tensor<6x6xi32> + %output2 = arith.constant dense<0> : tensor<6x6xi32> + %2 = call @conv2d_bs(%input_DCSR, %filter_CSR, %output2) + : (tensor<10x10xi32, #DCSR>, tensor<5x5xi32, #CSR>, tensor<6x6xi32>) -> tensor<6x6xi32> + + // Call the kernels with sparse input. + %output3 = arith.constant dense<0> : tensor<6x6xi32> + %3 = call @conv2d(%sinput, %sfilter, %output3) + : (tensor<10x10xi32>, tensor<5x5xi32>, tensor<6x6xi32>) -> tensor<6x6xi32> + %output4 = arith.constant dense<0> : tensor<6x6xi32> + %4 = call @conv2d_ss(%sinput_CSR, %sfilter_CSR, %output4) + : (tensor<10x10xi32, #CSR>, tensor<5x5xi32, #CSR>, tensor<6x6xi32>) -> tensor<6x6xi32> + %output5 = arith.constant dense<0> : tensor<6x6xi32> + %5 = call @conv2d_bs(%sinput_DCSR, %sfilter_CSR, %output5) + : (tensor<10x10xi32, #DCSR>, tensor<5x5xi32, #CSR>, tensor<6x6xi32>) -> tensor<6x6xi32> + + // Verify the output. + // + // CHECK: ( ( -9700, -10025, -10350, -10675, -11000, -11325 ), + // CHECK-SAME: ( -12950, -13275, -13600, -13925, -14250, -14575 ), + // CHECK-SAME: ( -16200, -16525, -16850, -17175, -17500, -17825 ), + // CHECK-SAME: ( -19450, -19775, -20100, -20425, -20750, -21075 ), + // CHECK-SAME: ( -22700, -23025, -23350, -23675, -24000, -24325 ), + // CHECK-SAME: ( -25950, -26275, -26600, -26925, -27250, -27575 ) ) + // + // CHECK: ( ( -9700, -10025, -10350, -10675, -11000, -11325 ), + // CHECK-SAME: ( -12950, -13275, -13600, -13925, -14250, -14575 ), + // CHECK-SAME: ( -16200, -16525, -16850, -17175, -17500, -17825 ), + // CHECK-SAME: ( -19450, -19775, -20100, -20425, -20750, -21075 ), + // CHECK-SAME: ( -22700, -23025, -23350, -23675, -24000, -24325 ), + // CHECK-SAME: ( -25950, -26275, -26600, -26925, -27250, -27575 ) ) + // + // CHECK: ( ( -9700, -10025, -10350, -10675, -11000, -11325 ), + // CHECK-SAME: ( -12950, -13275, -13600, -13925, -14250, -14575 ), + // CHECK-SAME: ( -16200, -16525, -16850, -17175, -17500, -17825 ), + // CHECK-SAME: ( -19450, -19775, -20100, -20425, -20750, -21075 ), + // CHECK-SAME: ( -22700, -23025, -23350, -23675, -24000, -24325 ), + // CHECK-SAME: ( -25950, -26275, -26600, -26925, -27250, -27575 ) ) + // + // CHECK: ( ( -7, -2, -39, 0, -30, -42 ), + // CHECK-SAME: ( -4, -10, 0, -77, 0, -40 ), + // CHECK-SAME: ( 0, 0, 0, 0, 16, 0 ), + // CHECK-SAME: ( 0, 0, 0, 0, 0, 64 ), + // CHECK-SAME: ( 0, 0, 0, -12, 0, -6 ), + // CHECK-SAME: ( -60, -27, -50, 0, -16, 0 ) ) + // + // CHECK: ( ( -7, -2, -39, 0, -30, -42 ), + // CHECK-SAME: ( -4, -10, 0, -77, 0, -40 ), + // CHECK-SAME: ( 0, 0, 0, 0, 16, 0 ), + // CHECK-SAME: ( 0, 0, 0, 0, 0, 64 ), + // CHECK-SAME: ( 0, 0, 0, -12, 0, -6 ), + // CHECK-SAME: ( -60, -27, -50, 0, -16, 0 ) ) + // + // CHECK: ( ( -7, -2, -39, 0, -30, -42 ), + // CHECK-SAME: ( -4, -10, 0, -77, 0, -40 ), + // CHECK-SAME: ( 0, 0, 0, 0, 16, 0 ), + // CHECK-SAME: ( 0, 0, 0, 0, 0, 64 ), + // CHECK-SAME: ( 0, 0, 0, -12, 0, -6 ), + // CHECK-SAME: ( -60, -27, -50, 0, -16, 0 ) ) + // + %v0 = vector.transfer_read %0[%c0, %c0], %i0 : tensor<6x6xi32>, vector<6x6xi32> + vector.print %v0 : vector<6x6xi32> + %v1 = vector.transfer_read %1[%c0, %c0], %i0 : tensor<6x6xi32>, vector<6x6xi32> + vector.print %v1 : vector<6x6xi32> + %v2 = vector.transfer_read %2[%c0, %c0], %i0 : tensor<6x6xi32>, vector<6x6xi32> + vector.print %v2 : vector<6x6xi32> + %v3 = vector.transfer_read %3[%c0, %c0], %i0 : tensor<6x6xi32>, vector<6x6xi32> + vector.print %v3 : vector<6x6xi32> + %v4 = vector.transfer_read %4[%c0, %c0], %i0 : tensor<6x6xi32>, vector<6x6xi32> + vector.print %v4 : vector<6x6xi32> + %v5 = vector.transfer_read %5[%c0, %c0], %i0 : tensor<6x6xi32>, vector<6x6xi32> + vector.print %v5 : vector<6x6xi32> + + // Release sparse resources. + bufferization.dealloc_tensor %input_CSR : tensor<10x10xi32, #CSR> + bufferization.dealloc_tensor %input_DCSR : tensor<10x10xi32, #DCSR> + bufferization.dealloc_tensor %filter_CSR : tensor<5x5xi32, #CSR> + bufferization.dealloc_tensor %sinput_CSR : tensor<10x10xi32, #CSR> + bufferization.dealloc_tensor %sinput_DCSR : tensor<10x10xi32, #DCSR> + bufferization.dealloc_tensor %sfilter_CSR : tensor<5x5xi32, #CSR> + + return + } +} diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir index e2138207f47e..c7e4ffab7ab3 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_2d_nchw_fchw.mlir @@ -25,7 +25,6 @@ // // Do the same run, but now with direct IR generation and vectorization. // REDEFINE: %{sparsifier_opts} = enable-runtime-library=false enable-buffer-initialization=true vl=2 reassociate-fp-reductions=true enable-index-optimizations=true - // RUN: %{compile} | %{run} | FileCheck %s // // Do the same run, but now with direct IR generation and VLA vectorization. diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir index dfb1bb71a68c..451d2b876946 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d.mlir @@ -38,10 +38,14 @@ map = (d0, d1, d2) -> (d0 : compressed, d1 : dense, d2 : compressed) }> -#DDC = #sparse_tensor.encoding<{ +#DCC = #sparse_tensor.encoding<{ map = (d0, d1, d2) -> (d0 : dense, d1 : compressed, d2 : compressed) }> +#DDC = #sparse_tensor.encoding<{ + map = (d0, d1, d2) -> (d0 : dense, d1 : dense, d2 : compressed) +}> + // Creates and returns 3-D buffer of size (%s1, %s2, %s3) filled with the value %f func.func @alloc_3d_filled_f32(%s1 : index, %s2 : index, %s3 : index, %f : f32) -> tensor { %buf = tensor.empty(%s1, %s2, %s3) : tensor @@ -74,6 +78,15 @@ func.func @conv_3d_CDC(%arg0: tensor, %arg1: tensor) return %ret : tensor } +func.func @conv_3d_DCC(%arg0: tensor, %arg1: tensor) -> tensor { + %c6 = arith.constant 6 : index + %s = tensor.empty(%c6, %c6, %c6) : tensor + %ret = linalg.conv_3d + ins (%arg0, %arg1: tensor, tensor) + outs (%s: tensor) -> tensor + return %ret : tensor +} + func.func @conv_3d_DDC(%arg0: tensor, %arg1: tensor) -> tensor { %c6 = arith.constant 6 : index %s = tensor.empty(%c6, %c6, %c6) : tensor @@ -102,12 +115,15 @@ func.func @entry() { : tensor to tensor %in3D_CDC = sparse_tensor.convert %in3D : tensor to tensor + %in3D_DCC = sparse_tensor.convert %in3D + : tensor to tensor %in3D_DDC = sparse_tensor.convert %in3D : tensor to tensor %dense_ret = call @conv_3d(%in3D, %filter3D, %out3D) : (tensor, tensor, tensor) -> (tensor) %CCC_ret = call @conv_3d_CCC(%in3D_CCC, %filter3D) : (tensor, tensor) -> (tensor) %CDC_ret = call @conv_3d_CDC(%in3D_CDC, %filter3D) : (tensor, tensor) -> (tensor) + %DCC_ret = call @conv_3d_DCC(%in3D_DCC, %filter3D) : (tensor, tensor) -> (tensor) %DDC_ret = call @conv_3d_DDC(%in3D_DDC, %filter3D) : (tensor, tensor) -> (tensor) // CHECK:( ( ( 108, 108, 108, 108, 108, 108 ), @@ -276,6 +292,48 @@ func.func @entry() { : tensor, vector<6x6x6xf32> vector.print %v2 : vector<6x6x6xf32> + // CHECK-NEXT:( ( ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 124, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 124, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 124, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ) ), + // CHECK-SAME: ( ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ) ), + // CHECK-SAME: ( ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ) ), + // CHECK-SAME: ( ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ) ), + // CHECK-SAME: ( ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ) ), + // CHECK-SAME: ( ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ), + // CHECK-SAME: ( 108, 108, 108, 108, 108, 108 ) ) ) + %4 = sparse_tensor.convert %DCC_ret + : tensor to tensor + %v4 = vector.transfer_read %3[%c0, %c0, %c0], %zero + : tensor, vector<6x6x6xf32> + vector.print %v2 : vector<6x6x6xf32> + // Free the resources bufferization.dealloc_tensor %in3D : tensor bufferization.dealloc_tensor %filter3D : tensor @@ -284,9 +342,11 @@ func.func @entry() { bufferization.dealloc_tensor %in3D_CDC : tensor bufferization.dealloc_tensor %in3D_CCC : tensor bufferization.dealloc_tensor %in3D_DDC : tensor + bufferization.dealloc_tensor %in3D_DCC : tensor bufferization.dealloc_tensor %CCC_ret : tensor bufferization.dealloc_tensor %CDC_ret : tensor bufferization.dealloc_tensor %DDC_ret : tensor + bufferization.dealloc_tensor %DCC_ret : tensor return } diff --git a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir index e96ee4d063e7..2c4b96804ccc 100644 --- a/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir +++ b/mlir/test/Integration/Dialect/SparseTensor/CPU/sparse_conv_3d_ndhwc_dhwcf.mlir @@ -25,7 +25,6 @@ // // Do the same run, but now with direct IR generation and vectorization. // REDEFINE: %{sparsifier_opts} = enable-runtime-library=false enable-buffer-initialization=true vl=2 reassociate-fp-reductions=true enable-index-optimizations=true - // RUN: %{compile} | %{run} | FileCheck %s // // Do the same run, but now with direct IR generation and VLA vectorization. diff --git a/mlir/test/Target/LLVMIR/Import/nsw_nuw.ll b/mlir/test/Target/LLVMIR/Import/nsw_nuw.ll new file mode 100644 index 000000000000..d08098a5e5df --- /dev/null +++ b/mlir/test/Target/LLVMIR/Import/nsw_nuw.ll @@ -0,0 +1,14 @@ +; RUN: mlir-translate -import-llvm -split-input-file %s | FileCheck %s + +; CHECK-LABEL: @intflag_inst +define void @intflag_inst(i64 %arg1, i64 %arg2) { + ; CHECK: llvm.add %{{.*}}, %{{.*}} overflow : i64 + %1 = add nsw i64 %arg1, %arg2 + ; CHECK: llvm.sub %{{.*}}, %{{.*}} overflow : i64 + %2 = sub nuw i64 %arg1, %arg2 + ; CHECK: llvm.mul %{{.*}}, %{{.*}} overflow : i64 + %3 = mul nsw nuw i64 %arg1, %arg2 + ; CHECK: llvm.shl %{{.*}}, %{{.*}} overflow : i64 + %4 = shl nuw nsw i64 %arg1, %arg2 + ret void +} diff --git a/mlir/test/Target/LLVMIR/nsw_nuw.mlir b/mlir/test/Target/LLVMIR/nsw_nuw.mlir new file mode 100644 index 000000000000..6843c2ef0299 --- /dev/null +++ b/mlir/test/Target/LLVMIR/nsw_nuw.mlir @@ -0,0 +1,14 @@ +// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s + +// CHECK-LABEL: define void @intflags_func +llvm.func @intflags_func(%arg0: i64, %arg1: i64) { + // CHECK: %{{.*}} = add nsw i64 %{{.*}}, %{{.*}} + %0 = llvm.add %arg0, %arg1 overflow : i64 + // CHECK: %{{.*}} = sub nuw i64 %{{.*}}, %{{.*}} + %1 = llvm.sub %arg0, %arg1 overflow : i64 + // CHECK: %{{.*}} = mul nuw nsw i64 %{{.*}}, %{{.*}} + %2 = llvm.mul %arg0, %arg1 overflow : i64 + // CHECK: %{{.*}} = shl nuw nsw i64 %{{.*}}, %{{.*}} + %3 = llvm.shl %arg0, %arg1 overflow : i64 + llvm.return +} diff --git a/mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp b/mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp index 565d07669792..a8a808424b69 100644 --- a/mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp +++ b/mlir/test/lib/Dialect/SCF/TestSCFUtils.cpp @@ -217,6 +217,7 @@ struct TestSCFPipeliningPass if (annotatePipeline) options.annotateFn = annotate; if (noEpiloguePeeling) { + options.supportDynamicLoops = true; options.peelEpilogue = false; options.predicateFn = predicateOp; } diff --git a/mlir/test/lib/Dialect/Test/TestOps.td b/mlir/test/lib/Dialect/Test/TestOps.td index 1add9bd3c329..96f66c2ca06e 100644 --- a/mlir/test/lib/Dialect/Test/TestOps.td +++ b/mlir/test/lib/Dialect/Test/TestOps.td @@ -2020,6 +2020,20 @@ def AffineScopeOp : TEST_Op<"affine_scope", [AffineScope]> { let hasCustomAssemblyFormat = 1; } +//===----------------------------------------------------------------------===// +// Custom printer/parser + +def CustomDimensionListAttrOp : TEST_Op<"custom_dimension_list_attr"> { + let description = [{ + Test printing/parsing of dimension list attribute. + }]; + let arguments = (ins DenseI64ArrayAttr:$dimension_list); + let assemblyFormat = [{ + `dimension_list` `=` custom($dimension_list) + attr-dict + }]; +} + //===----------------------------------------------------------------------===// // Test OpAsmInterface. diff --git a/mlir/test/lib/Transforms/TestIntRangeInference.cpp b/mlir/test/lib/Transforms/TestIntRangeInference.cpp index d1978b6099f0..2f6dd5b8095d 100644 --- a/mlir/test/lib/Transforms/TestIntRangeInference.cpp +++ b/mlir/test/lib/Transforms/TestIntRangeInference.cpp @@ -9,6 +9,7 @@ // functionality has been integrated into SCCP. //===----------------------------------------------------------------------===// +#include "mlir/Analysis/DataFlow/ConstantPropagationAnalysis.h" #include "mlir/Analysis/DataFlow/DeadCodeAnalysis.h" #include "mlir/Analysis/DataFlow/IntegerRangeAnalysis.h" #include "mlir/Interfaces/SideEffectInterfaces.h" @@ -107,6 +108,7 @@ struct TestIntRangeInference Operation *op = getOperation(); DataFlowSolver solver; solver.load(); + solver.load(); solver.load(); if (failed(solver.initializeAndRun(op))) return signalPassFailure(); diff --git a/mlir/test/python/dialects/affine.py b/mlir/test/python/dialects/affine.py index df42f8fcf1a5..6f39e1348fcd 100644 --- a/mlir/test/python/dialects/affine.py +++ b/mlir/test/python/dialects/affine.py @@ -5,6 +5,7 @@ from mlir.dialects import func from mlir.dialects import arith from mlir.dialects import memref from mlir.dialects import affine +import mlir.extras.types as T def constructAndPrintInModule(f): @@ -43,6 +44,17 @@ def testAffineStoreOp(): return mem +# CHECK-LABEL: TEST: testAffineDelinearizeInfer +@constructAndPrintInModule +def testAffineDelinearizeInfer(): + # CHECK: %[[C0:.*]] = arith.constant 0 : index + c0 = arith.ConstantOp(T.index(), 0) + # CHECK: %[[C1:.*]] = arith.constant 1 : index + c1 = arith.ConstantOp(T.index(), 1) + # CHECK: %{{.*}}:2 = affine.delinearize_index %[[C1:.*]] into (%[[C1:.*]], %[[C0:.*]]) : index, index + two_indices = affine.AffineDelinearizeIndexOp(c1, [c1, c0]) + + # CHECK-LABEL: TEST: testAffineLoadOp @constructAndPrintInModule def testAffineLoadOp(): @@ -107,66 +119,149 @@ def testAffineForOp(): # CHECK: %[[TMP:.*]] = memref.load %[[BUFFER]][%[[INDVAR]]] : memref<1024xf32> tmp = memref.LoadOp(buffer, [sum.induction_variable]) sum_next = arith.AddFOp(sum.inner_iter_args[0], tmp) - affine.AffineYieldOp([sum_next]) - return + +# CHECK-LABEL: TEST: testAffineForOpErrors +@constructAndPrintInModule +def testAffineForOpErrors(): + c1 = arith.ConstantOp(T.index(), 1) + c2 = arith.ConstantOp(T.index(), 2) + c3 = arith.ConstantOp(T.index(), 3) + d0 = AffineDimExpr.get(0) + + try: + affine.AffineForOp( + c1, + c2, + 1, + lower_bound_operands=[c3], + upper_bound_operands=[], + ) + except ValueError as e: + assert ( + e.args[0] + == "Either a concrete lower bound or an AffineMap in combination with lower bound operands, but not both, is supported." + ) + + try: + affine.AffineForOp( + AffineMap.get_constant(1), + c2, + 1, + lower_bound_operands=[c3, c3], + upper_bound_operands=[], + ) + except ValueError as e: + assert ( + e.args[0] + == "Wrong number of lower bound operands passed to AffineForOp; Expected 0, got 2." + ) + + try: + two_indices = affine.AffineDelinearizeIndexOp(c1, [c1, c1]) + affine.AffineForOp( + two_indices, + c2, + 1, + lower_bound_operands=[], + upper_bound_operands=[], + ) + except ValueError as e: + assert e.args[0] == "Only a single concrete value is supported for lower bound." + + try: + affine.AffineForOp( + 1.0, + c2, + 1, + lower_bound_operands=[], + upper_bound_operands=[], + ) + except ValueError as e: + assert e.args[0] == "lower bound must be int | ResultValueT | AffineMap." @constructAndPrintInModule def testForSugar(): - index_type = IndexType.get() - memref_t = MemRefType.get([10], index_type) + memref_t = T.memref(10, T.index()) range = affine.for_ - # CHECK: func.func @range_loop_1(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index, %[[VAL_3:.*]]: memref<10xindex>) { - # CHECK: %[[VAL_4:.*]] = arith.constant 10 : index - # CHECK: affine.for %[[VAL_6:.*]] = %[[VAL_0]] to %[[VAL_4]] step 2 { - # CHECK: %[[VAL_7:.*]] = arith.addi %[[VAL_6]], %[[VAL_6]] : index - # CHECK: affine.store %[[VAL_7]], %[[VAL_3]]{{\[symbol\(}}%[[VAL_6]]{{\)\]}} : memref<10xindex> - # CHECK: } - # CHECK: return - # CHECK: } - @func.FuncOp.from_py_func(index_type, index_type, index_type, memref_t) - def range_loop_1(lb, ub, step, memref_v): - for i in range(lb, 10, 2): + # CHECK: #[[$ATTR_2:.+]] = affine_map<(d0) -> (d0)> + + # CHECK-LABEL: func.func @range_loop_1( + # CHECK-SAME: %[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: memref<10xindex>) { + # CHECK: affine.for %[[VAL_3:.*]] = #[[$ATTR_2]](%[[VAL_0]]) to #[[$ATTR_2]](%[[VAL_1]]) { + # CHECK: %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_3]] : index + # CHECK: memref.store %[[VAL_4]], %[[VAL_2]]{{\[}}%[[VAL_3]]] : memref<10xindex> + # CHECK: } + # CHECK: return + # CHECK: } + @func.FuncOp.from_py_func(T.index(), T.index(), memref_t) + def range_loop_1(lb, ub, memref_v): + for i in range(lb, ub, step=1): + add = arith.addi(i, i) + memref.store(add, memref_v, [i]) + + affine.yield_([]) + + # CHECK-LABEL: func.func @range_loop_2( + # CHECK-SAME: %[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: memref<10xindex>) { + # CHECK: affine.for %[[VAL_3:.*]] = #[[$ATTR_2]](%[[VAL_0]]) to 10 { + # CHECK: %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_3]] : index + # CHECK: memref.store %[[VAL_4]], %[[VAL_2]]{{\[}}%[[VAL_3]]] : memref<10xindex> + # CHECK: } + # CHECK: return + # CHECK: } + @func.FuncOp.from_py_func(T.index(), T.index(), memref_t) + def range_loop_2(lb, ub, memref_v): + for i in range(lb, 10, step=1): + add = arith.addi(i, i) + memref.store(add, memref_v, [i]) + affine.yield_([]) + + # CHECK-LABEL: func.func @range_loop_3( + # CHECK-SAME: %[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: memref<10xindex>) { + # CHECK: affine.for %[[VAL_3:.*]] = 0 to #[[$ATTR_2]](%[[VAL_1]]) { + # CHECK: %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_3]] : index + # CHECK: memref.store %[[VAL_4]], %[[VAL_2]]{{\[}}%[[VAL_3]]] : memref<10xindex> + # CHECK: } + # CHECK: return + # CHECK: } + @func.FuncOp.from_py_func(T.index(), T.index(), memref_t) + def range_loop_3(lb, ub, memref_v): + for i in range(0, ub, step=1): add = arith.addi(i, i) - s0 = AffineSymbolExpr.get(0) - map = AffineMap.get(0, 1, [s0]) - affine.store(add, memref_v, [i], map=map) - affine.AffineYieldOp([]) - - # CHECK: func.func @range_loop_2(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index, %[[VAL_3:.*]]: memref<10xindex>) { - # CHECK: %[[VAL_4:.*]] = arith.constant 0 : index - # CHECK: %[[VAL_5:.*]] = arith.constant 10 : index - # CHECK: affine.for %[[VAL_7:.*]] = %[[VAL_4]] to %[[VAL_5]] { - # CHECK: %[[VAL_8:.*]] = arith.addi %[[VAL_7]], %[[VAL_7]] : index - # CHECK: affine.store %[[VAL_8]], %[[VAL_3]]{{\[symbol\(}}%[[VAL_7]]{{\)\]}} : memref<10xindex> - # CHECK: } - # CHECK: return - # CHECK: } - @func.FuncOp.from_py_func(index_type, index_type, index_type, memref_t) - def range_loop_2(lb, ub, step, memref_v): - for i in range(0, 10, 1): + memref.store(add, memref_v, [i]) + affine.yield_([]) + + # CHECK-LABEL: func.func @range_loop_4( + # CHECK-SAME: %[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: memref<10xindex>) { + # CHECK: affine.for %[[VAL_3:.*]] = 0 to 10 { + # CHECK: %[[VAL_4:.*]] = arith.addi %[[VAL_3]], %[[VAL_3]] : index + # CHECK: memref.store %[[VAL_4]], %[[VAL_2]]{{\[}}%[[VAL_3]]] : memref<10xindex> + # CHECK: } + # CHECK: return + # CHECK: } + @func.FuncOp.from_py_func(T.index(), T.index(), memref_t) + def range_loop_4(lb, ub, memref_v): + for i in range(0, 10, step=1): add = arith.addi(i, i) - s0 = AffineSymbolExpr.get(0) - map = AffineMap.get(0, 1, [s0]) - affine.store(add, memref_v, [i], map=map) - affine.AffineYieldOp([]) - - # CHECK: func.func @range_loop_3(%[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: index, %[[VAL_3:.*]]: memref<10xindex>) { - # CHECK: %[[VAL_4:.*]] = arith.constant 0 : index - # CHECK: affine.for %[[VAL_6:.*]] = %[[VAL_4]] to %[[VAL_1]] { - # CHECK: %[[VAL_7:.*]] = arith.addi %[[VAL_6]], %[[VAL_6]] : index - # CHECK: affine.store %[[VAL_7]], %[[VAL_3]]{{\[symbol\(}}%[[VAL_6]]{{\)\]}} : memref<10xindex> - # CHECK: } - # CHECK: return - # CHECK: } - @func.FuncOp.from_py_func(index_type, index_type, index_type, memref_t) - def range_loop_3(lb, ub, step, memref_v): - for i in range(0, ub, 1): + memref.store(add, memref_v, [i]) + affine.yield_([]) + + # CHECK-LABEL: func.func @range_loop_8( + # CHECK-SAME: %[[VAL_0:.*]]: index, %[[VAL_1:.*]]: index, %[[VAL_2:.*]]: memref<10xindex>) { + # CHECK: %[[VAL_3:.*]] = affine.for %[[VAL_4:.*]] = 0 to 10 iter_args(%[[VAL_5:.*]] = %[[VAL_2]]) -> (memref<10xindex>) { + # CHECK: %[[VAL_6:.*]] = arith.addi %[[VAL_4]], %[[VAL_4]] : index + # CHECK: memref.store %[[VAL_6]], %[[VAL_5]]{{\[}}%[[VAL_4]]] : memref<10xindex> + # CHECK: affine.yield %[[VAL_5]] : memref<10xindex> + # CHECK: } + # CHECK: return + # CHECK: } + @func.FuncOp.from_py_func(T.index(), T.index(), memref_t) + def range_loop_8(lb, ub, memref_v): + for i, it in range(0, 10, iter_args=[memref_v]): add = arith.addi(i, i) - s0 = AffineSymbolExpr.get(0) - map = AffineMap.get(0, 1, [s0]) - affine.store(add, memref_v, [i], map=map) - affine.AffineYieldOp([]) + memref.store(add, it, [i]) + affine.yield_([it]) diff --git a/mlir/unittests/Pass/CMakeLists.txt b/mlir/unittests/Pass/CMakeLists.txt index 65f077412386..802b3bbc6c63 100644 --- a/mlir/unittests/Pass/CMakeLists.txt +++ b/mlir/unittests/Pass/CMakeLists.txt @@ -5,5 +5,6 @@ add_mlir_unittest(MLIRPassTests ) target_link_libraries(MLIRPassTests PRIVATE + MLIRDebug MLIRFuncDialect MLIRPass) diff --git a/mlir/unittests/Pass/PassManagerTest.cpp b/mlir/unittests/Pass/PassManagerTest.cpp index 9a30f64eaabc..7ceed3bb3bc3 100644 --- a/mlir/unittests/Pass/PassManagerTest.cpp +++ b/mlir/unittests/Pass/PassManagerTest.cpp @@ -7,6 +7,8 @@ //===----------------------------------------------------------------------===// #include "mlir/Pass/PassManager.h" +#include "mlir/Debug/BreakpointManagers/TagBreakpointManager.h" +#include "mlir/Debug/ExecutionContext.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/IR/Builders.h" #include "mlir/IR/BuiltinOps.h" @@ -86,6 +88,104 @@ TEST(PassManagerTest, OpSpecificAnalysis) { } } +/// Simple pass to annotate a func::FuncOp with a single attribute `didProcess`. +struct AddAttrFunctionPass + : public PassWrapper> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(AddAttrFunctionPass) + + void runOnOperation() override { + func::FuncOp op = getOperation(); + Builder builder(op->getParentOfType()); + if (op->hasAttr("didProcess")) + op->setAttr("didProcessAgain", builder.getUnitAttr()); + + // We always want to set this one. + op->setAttr("didProcess", builder.getUnitAttr()); + } +}; + +/// Simple pass to annotate a func::FuncOp with a single attribute +/// `didProcess2`. +struct AddSecondAttrFunctionPass + : public PassWrapper> { + MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(AddSecondAttrFunctionPass) + + void runOnOperation() override { + func::FuncOp op = getOperation(); + Builder builder(op->getParentOfType()); + op->setAttr("didProcess2", builder.getUnitAttr()); + } +}; + +TEST(PassManagerTest, ExecutionAction) { + MLIRContext context; + context.loadDialect(); + Builder builder(&context); + + // Create a module with 2 functions. + OwningOpRef module(ModuleOp::create(UnknownLoc::get(&context))); + auto f = + func::FuncOp::create(builder.getUnknownLoc(), "process_me_once", + builder.getFunctionType(std::nullopt, std::nullopt)); + f.setPrivate(); + module->push_back(f); + + // Instantiate our passes. + auto pm = PassManager::on(&context); + auto pass = std::make_unique(); + auto *passPtr = pass.get(); + pm.addNestedPass(std::move(pass)); + pm.addNestedPass(std::make_unique()); + // Duplicate the first pass to ensure that we *only* run the *first* pass, not + // all instances of this pass kind. Notice that this pass (and the test as a + // whole) are built to ensure that we can run just a single pass out of a + // pipeline that may contain duplicates. + pm.addNestedPass(std::make_unique()); + + // Use the action manager to only hit the first pass, not the second one. + auto onBreakpoint = [&](const tracing::ActionActiveStack *backtrace) + -> tracing::ExecutionContext::Control { + // Not a PassExecutionAction, apply the action. + auto *passExec = dyn_cast(&backtrace->getAction()); + if (!passExec) + return tracing::ExecutionContext::Next; + + // If this isn't a function, apply the action. + if (!isa(passExec->getOp())) + return tracing::ExecutionContext::Next; + + // Only apply the first function pass. Not all instances of the first pass, + // only the first pass. + if (passExec->getPass().getThreadingSiblingOrThis() == passPtr) + return tracing::ExecutionContext::Next; + + // Do not apply any other passes in the pass manager. + return tracing::ExecutionContext::Skip; + }; + + // Set up our breakpoint manager. + tracing::TagBreakpointManager simpleManager; + tracing::ExecutionContext executionCtx(onBreakpoint); + executionCtx.addBreakpointManager(&simpleManager); + simpleManager.addBreakpoint(PassExecutionAction::tag); + + // Register the execution context in the MLIRContext. + context.registerActionHandler(executionCtx); + + // Run the pass manager, expecting our handler to be called. + LogicalResult result = pm.run(module.get()); + EXPECT_TRUE(succeeded(result)); + + // Verify that each function got annotated with `didProcess` and *not* + // `didProcess2`. + for (func::FuncOp func : module->getOps()) { + ASSERT_TRUE(func->getDiscardableAttr("didProcess")); + ASSERT_FALSE(func->getDiscardableAttr("didProcess2")); + ASSERT_FALSE(func->getDiscardableAttr("didProcessAgain")); + } +} + namespace { struct InvalidPass : Pass { MLIR_DEFINE_EXPLICIT_INTERNAL_INLINE_TYPE_ID(InvalidPass) diff --git a/openmp/libomptarget/CMakeLists.txt b/openmp/libomptarget/CMakeLists.txt index 115189a28ce1..66925ccbe030 100644 --- a/openmp/libomptarget/CMakeLists.txt +++ b/openmp/libomptarget/CMakeLists.txt @@ -24,7 +24,7 @@ set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${LIBOMPTARGET_LIBRARY_DIR}) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${LIBOMPTARGET_LIBRARY_DIR}) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${LIBOMPTARGET_LIBRARY_DIR}) -if(LLVM_LIBRARY_OUTPUT_INTDIR) +if(NOT LLVM_LIBRARY_OUTPUT_INTDIR) set(LIBOMPTARGET_INTDIR ${LIBOMPTARGET_LIBRARY_DIR}) else() set(LIBOMPTARGET_INTDIR ${LLVM_LIBRARY_OUTPUT_INTDIR}) @@ -75,6 +75,25 @@ if(LIBOMPTARGET_ENABLE_DEBUG) add_definitions(-DOMPTARGET_DEBUG) endif() +# No exceptions and no RTTI, except if requested. +set(offload_compile_flags -fno-exceptions) +if(NOT LLVM_ENABLE_RTTI) + set(offload_compile_flags ${offload_compile_flags} -fno-rtti) +endif() + +# If LTO is not explicitly disabled we check if we can enable it and do so. +set(LIBOMPTARGET_USE_LTO TRUE CACHE BOOL "Use LTO for the offload runtimes if available") +if (LIBOMPTARGET_USE_LTO) + include(CheckIPOSupported) + check_ipo_supported(RESULT use_lto OUTPUT output) + if(use_lto) + set(offload_compile_flags ${offload_compile_flags} -flto) + set(offload_link_flags ${offload_link_flags} -flto) + else() + message(WARNING "LTO is not supported: ${output}") + endif() +endif() + # OMPT support for libomptarget # Follow host OMPT support and check if host support has been requested. # LIBOMP_HAVE_OMPT_SUPPORT indicates whether host OMPT support has been implemented. diff --git a/openmp/libomptarget/DeviceRTL/CMakeLists.txt b/openmp/libomptarget/DeviceRTL/CMakeLists.txt index 104b65020daf..1ce3e1e40a80 100644 --- a/openmp/libomptarget/DeviceRTL/CMakeLists.txt +++ b/openmp/libomptarget/DeviceRTL/CMakeLists.txt @@ -302,7 +302,7 @@ endforeach() # Archive all the object files generated above into a static library add_library(omptarget.devicertl STATIC) set_target_properties(omptarget.devicertl PROPERTIES - ARCHIVE_OUTPUT_DIRECTORY ${LIBOMPTARGET_LLVM_LIBRARY_INTDIR} + ARCHIVE_OUTPUT_DIRECTORY "${LIBOMPTARGET_LLVM_LIBRARY_INTDIR}" LINKER_LANGUAGE CXX ) target_link_libraries(omptarget.devicertl PRIVATE omptarget.devicertl.all_objs) diff --git a/openmp/libomptarget/include/PluginManager.h b/openmp/libomptarget/include/PluginManager.h index 6af99efa32a0..0b0974709b52 100644 --- a/openmp/libomptarget/include/PluginManager.h +++ b/openmp/libomptarget/include/PluginManager.h @@ -150,6 +150,24 @@ struct PluginManager { HostPtrToTableMapTy HostPtrToTableMap; std::mutex TblMapMtx; ///< For HostPtrToTableMap + // Work around for plugins that call dlopen on shared libraries that call + // tgt_register_lib during their initialisation. Stash the pointers in a + // vector until the plugins are all initialised and then register them. + bool delayRegisterLib(__tgt_bin_desc *Desc) { + if (RTLsLoaded) + return false; + DelayedBinDesc.push_back(Desc); + return true; + } + + void registerDelayedLibraries() { + // Only called by libomptarget constructor + RTLsLoaded = true; + for (auto *Desc : DelayedBinDesc) + __tgt_register_lib(Desc); + DelayedBinDesc.clear(); + } + /// Return the number of usable devices. int getNumDevices() { return getExclusiveDevicesAccessor()->size(); } @@ -178,6 +196,9 @@ struct PluginManager { void addRequirements(int64_t Flags) { Requirements.addRequirements(Flags); } private: + bool RTLsLoaded = false; + llvm::SmallVector<__tgt_bin_desc *> DelayedBinDesc; + // List of all plugin adaptors, in use or not. llvm::SmallVector> PluginAdaptors; diff --git a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt index 5b332ed3d2f4..8ae3ff2a6d29 100644 --- a/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt +++ b/openmp/libomptarget/plugins-nextgen/common/CMakeLists.txt @@ -88,6 +88,9 @@ target_compile_definitions(PluginCommon PRIVATE DEBUG_PREFIX="PluginInterface" ) +target_compile_options(PluginCommon PUBLIC ${offload_compile_flags}) +target_link_options(PluginCommon PUBLIC ${offload_link_flags}) + target_include_directories(PluginCommon PRIVATE ${LIBOMPTARGET_INCLUDE_DIR} diff --git a/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp b/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp index a3d16d3a5bcf..0a19148ca4ec 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/GlobalHandler.cpp @@ -16,6 +16,10 @@ #include "Shared/Utils.h" +#include "llvm/BinaryFormat/ELF.h" +#include "llvm/Support/Error.h" + +#include #include using namespace llvm; @@ -53,9 +57,15 @@ Error GenericGlobalHandlerTy::getGlobalMetadataFromELF( const ELF64LE::Shdr &Section, GlobalTy &ImageGlobal) { // The global's address is computed as the image begin + the ELF section - // offset + the ELF symbol value. - ImageGlobal.setPtr(advanceVoidPtr( - Image.getStart(), Section.sh_offset - Section.sh_addr + Symbol.st_value)); + // offset + the ELF symbol value except for NOBITS sections that, as the name + // suggests, have no bits in the image. We still record the size and use + // nullptr to indicate there is no location. + if (Section.sh_type == ELF::SHT_NOBITS) + ImageGlobal.setPtr(nullptr); + else + ImageGlobal.setPtr( + advanceVoidPtr(Image.getStart(), + Section.sh_offset - Section.sh_addr + Symbol.st_value)); // Set the global's size. ImageGlobal.setSize(Symbol.st_size); @@ -170,12 +180,21 @@ Error GenericGlobalHandlerTy::readGlobalFromImage(GenericDeviceTy &Device, "%u bytes in the ELF image but %u bytes on the host", HostGlobal.getName().data(), ImageGlobal.getSize(), HostGlobal.getSize()); + if (ImageGlobal.getPtr() == nullptr) + return Plugin::error("Transfer impossible because global symbol '%s' has " + "no representation in the image (NOBITS sections)", + HostGlobal.getName().data()); DP("Global symbol '%s' was found in the ELF image and %u bytes will copied " "from %p to %p.\n", HostGlobal.getName().data(), HostGlobal.getSize(), ImageGlobal.getPtr(), HostGlobal.getPtr()); + assert(Image.getStart() <= ImageGlobal.getPtr() && + advanceVoidPtr(ImageGlobal.getPtr(), ImageGlobal.getSize()) < + advanceVoidPtr(Image.getStart(), Image.getSize()) && + "Attempting to read outside the image!"); + // Perform the copy from the image to the host memory. std::memcpy(HostGlobal.getPtr(), ImageGlobal.getPtr(), HostGlobal.getSize()); diff --git a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp index 912e3d2c479b..3c7d1ca89987 100644 --- a/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp +++ b/openmp/libomptarget/plugins-nextgen/common/src/PluginInterface.cpp @@ -785,9 +785,14 @@ Error GenericDeviceTy::deinit(GenericPluginTy &Plugin) { GenericGlobalHandlerTy &GHandler = Plugin.getGlobalHandler(); for (auto *Image : LoadedImages) { DeviceMemoryPoolTrackingTy ImageDeviceMemoryPoolTracking = {0, 0, ~0U, 0}; - if (!GHandler.isSymbolInImage(*this, *Image, - "__omp_rtl_device_memory_pool_tracker")) + GlobalTy TrackerGlobal("__omp_rtl_device_memory_pool_tracker", + sizeof(DeviceMemoryPoolTrackingTy), + &ImageDeviceMemoryPoolTracking); + if (auto Err = + GHandler.readGlobalFromDevice(*this, *Image, TrackerGlobal)) { + consumeError(std::move(Err)); continue; + } DeviceMemoryPoolTracking.combine(ImageDeviceMemoryPoolTracking); } @@ -968,16 +973,16 @@ Error GenericDeviceTy::setupDeviceMemoryPool(GenericPluginTy &Plugin, } // Create the metainfo of the device environment global. - GlobalTy TrackerGlobal("__omp_rtl_device_memory_pool_tracker", - sizeof(DeviceMemoryPoolTrackingTy), - &DeviceMemoryPoolTracking); GenericGlobalHandlerTy &GHandler = Plugin.getGlobalHandler(); - if (auto Err = GHandler.readGlobalFromImage(*this, Image, TrackerGlobal)) { - [[maybe_unused]] std::string ErrStr = toString(std::move(Err)); - DP("Avoid the memory pool: %s.\n", ErrStr.c_str()); + if (!GHandler.isSymbolInImage(*this, Image, + "__omp_rtl_device_memory_pool_tracker")) { + DP("Skip the memory pool as there is no tracker symbol in the image."); return Error::success(); } + GlobalTy TrackerGlobal("__omp_rtl_device_memory_pool_tracker", + sizeof(DeviceMemoryPoolTrackingTy), + &DeviceMemoryPoolTracking); if (auto Err = GHandler.writeGlobalToDevice(*this, Image, TrackerGlobal)) return Err; diff --git a/openmp/libomptarget/src/CMakeLists.txt b/openmp/libomptarget/src/CMakeLists.txt index 7c07c61142ba..429a670be849 100644 --- a/openmp/libomptarget/src/CMakeLists.txt +++ b/openmp/libomptarget/src/CMakeLists.txt @@ -55,6 +55,9 @@ target_compile_definitions(omptarget PRIVATE DEBUG_PREFIX="omptarget" ) +target_compile_options(omptarget PUBLIC ${offload_compile_flags}) +target_link_options(omptarget PUBLIC ${offload_link_flags}) + macro(check_plugin_target target) if (TARGET omptarget.rtl.${target}) list(APPEND LIBOMPTARGET_PLUGINS_TO_LOAD ${target}) diff --git a/openmp/libomptarget/src/PluginManager.cpp b/openmp/libomptarget/src/PluginManager.cpp index f93e88e9e274..16e04dde923d 100644 --- a/openmp/libomptarget/src/PluginManager.cpp +++ b/openmp/libomptarget/src/PluginManager.cpp @@ -61,7 +61,7 @@ Error PluginAdaptorTy::init() { return createStringError(inconvertibleErrorCode(), \ "Invalid plugin as necessary interface function " \ "(%s) was not found.\n", \ - NAME); \ + std::string(#NAME).c_str()); \ } #include "Shared/PluginAPI.inc" diff --git a/openmp/libomptarget/src/interface.cpp b/openmp/libomptarget/src/interface.cpp index 777dc07943aa..d92f40ce1d14 100644 --- a/openmp/libomptarget/src/interface.cpp +++ b/openmp/libomptarget/src/interface.cpp @@ -46,6 +46,9 @@ EXTERN void __tgt_register_requires(int64_t Flags) { /// adds a target shared library to the target execution image EXTERN void __tgt_register_lib(__tgt_bin_desc *Desc) { TIMESCOPE(); + if (PM->delayRegisterLib(Desc)) + return; + PM->registerLib(Desc); } diff --git a/openmp/libomptarget/src/rtl.cpp b/openmp/libomptarget/src/rtl.cpp index 27db7034d895..5eb1c553df49 100644 --- a/openmp/libomptarget/src/rtl.cpp +++ b/openmp/libomptarget/src/rtl.cpp @@ -51,6 +51,7 @@ __attribute__((constructor(101))) void init() { PM->init(); Profiler::get(); + PM->registerDelayedLibraries(); } __attribute__((destructor(101))) void deinit() { diff --git a/openmp/libomptarget/test/Inputs/empty.c b/openmp/libomptarget/test/Inputs/empty.c deleted file mode 100644 index 8b137891791f..000000000000 --- a/openmp/libomptarget/test/Inputs/empty.c +++ /dev/null @@ -1 +0,0 @@ - diff --git a/openmp/libomptarget/test/offloading/back2back_distribute.c b/openmp/libomptarget/test/offloading/back2back_distribute.c index 750e39061974..63cabd0c6678 100644 --- a/openmp/libomptarget/test/offloading/back2back_distribute.c +++ b/openmp/libomptarget/test/offloading/back2back_distribute.c @@ -15,12 +15,12 @@ void reset_input(double *a, double *a_h, double *b, double *c) { } int main(int argc, char *argv[]) { - double * a = (double *) malloc(MAX_N * sizeof(double)); - double * a_h = (double *) malloc(MAX_N * sizeof(double)); - double * d = (double *) malloc(MAX_N * sizeof(double)); - double * d_h = (double *) malloc(MAX_N * sizeof(double)); - double * b = (double *) malloc(MAX_N * sizeof(double)); - double * c = (double *) malloc(MAX_N * sizeof(double)); + double *a = (double *)calloc(MAX_N, sizeof(double)); + double *a_h = (double *)calloc(MAX_N, sizeof(double)); + double *d = (double *)calloc(MAX_N, sizeof(double)); + double *d_h = (double *)calloc(MAX_N, sizeof(double)); + double *b = (double *)calloc(MAX_N, sizeof(double)); + double *c = (double *)calloc(MAX_N, sizeof(double)); #pragma omp target enter data map(to:a[:MAX_N],b[:MAX_N],c[:MAX_N],d[:MAX_N]) diff --git a/openmp/libomptarget/test/offloading/barrier_fence.c b/openmp/libomptarget/test/offloading/barrier_fence.c index a0b672fb1a84..5d1096478ed9 100644 --- a/openmp/libomptarget/test/offloading/barrier_fence.c +++ b/openmp/libomptarget/test/offloading/barrier_fence.c @@ -7,8 +7,6 @@ // UNSUPPORTED: aarch64-unknown-linux-gnu-LTO // UNSUPPORTED: x86_64-pc-linux-gnu // UNSUPPORTED: x86_64-pc-linux-gnu-LTO -// UNSUPPORTED: amdgcn-amd-amdhsa -// UNSUPPORTED: amdgcn-amd-amdhsa-LTO #include #include diff --git a/openmp/libomptarget/test/offloading/bug60119.c b/openmp/libomptarget/test/offloading/bug60119.c deleted file mode 100644 index e32f1ccd5988..000000000000 --- a/openmp/libomptarget/test/offloading/bug60119.c +++ /dev/null @@ -1,6 +0,0 @@ -// RUN: %clang-generic -fPIC -shared %S/../Inputs/empty.c -o %T/liba.so -// RUN: %clang-generic -fPIC -shared %S/../Inputs/empty.c -o %T/libb.so -// RUN: %clang-generic -rpath %T -L %T -l a -l b %s -o %t -// RUN: %t - -int main() {} diff --git a/openmp/libomptarget/test/offloading/bug74582.c b/openmp/libomptarget/test/offloading/bug74582.c new file mode 100644 index 000000000000..c6a283bb9369 --- /dev/null +++ b/openmp/libomptarget/test/offloading/bug74582.c @@ -0,0 +1,13 @@ +// RUN: %libomptarget-compile-generic && %libomptarget-run-generic +// RUN: %libomptarget-compileopt-generic && %libomptarget-run-generic + +// Verify we do not read bits in the image that are not there (nobits section). + +#pragma omp begin declare target +char BigUninitializedBuffer[4096 * 64] __attribute__((loader_uninitialized)); +#pragma omp end declare target + +int main() { +#pragma omp target + {} +} diff --git a/openmp/libomptarget/test/offloading/shared_lib_fp_mapping.c b/openmp/libomptarget/test/offloading/shared_lib_fp_mapping.c index a98a84ccceaa..ffb4bc90d4e4 100644 --- a/openmp/libomptarget/test/offloading/shared_lib_fp_mapping.c +++ b/openmp/libomptarget/test/offloading/shared_lib_fp_mapping.c @@ -1,6 +1,6 @@ // clang-format off -// RUN: %clang-generic -fPIC -shared %S/../Inputs/declare_indirect_func.c -o %T/liba.so -fopenmp-version=51 -// RUN: %libomptarget-compile-generic -rpath %T -L %T -l a -o %t -fopenmp-version=51 +// RUN: %clang-generic -fPIC -shared %S/../Inputs/declare_indirect_func.c -o %T/libslfm.so -fopenmp-version=51 +// RUN: %libomptarget-compile-generic -rpath %T -L %T -l slfm -o %t -fopenmp-version=51 // RUN: env LIBOMPTARGET_INFO=32 %t 2>&1 | %fcheck-generic // clang-format on diff --git a/openmp/libomptarget/test/offloading/target_map_for_member_data.cpp b/openmp/libomptarget/test/offloading/target_map_for_member_data.cpp new file mode 100644 index 000000000000..7fba71be6e66 --- /dev/null +++ b/openmp/libomptarget/test/offloading/target_map_for_member_data.cpp @@ -0,0 +1,25 @@ +// clang-format off +// RUN: %libomptarget-compilexx-generic && env LIBOMPTARGET_DEBUG=1 %libomptarget-run-generic 2>&1 | %fcheck-generic +// clang-format on + +// REQUIRES: libomptarget-debug + +struct DataTy { + float a; + float b[3]; +}; + +int main(int argc, char **argv) { + DataTy D; +#pragma omp target map(D.a) map(D.b[ : 2]) + { + D.a = 0; + D.b[0] = 1; + } + return 0; +} +// clang-format off +// CHECK: omptarget --> Entry 0: Base=[[DAT_HST_PTR_BASE:0x.*]], Begin=[[DAT_HST_PTR_BASE]], Size=12 +// CHECK: omptarget --> Entry 1: Base=[[DAT_HST_PTR_BASE]], Begin=[[DAT_HST_PTR_BASE]], Size=4, +// CHECK: omptarget --> Entry 2: Base=[[DAT_HST_PTR_BASE]], Begin=[[DATUM_HST_PTR_BASE:0x.*]], Size=8, +// clang-format on diff --git a/openmp/runtime/src/z_Linux_util.cpp b/openmp/runtime/src/z_Linux_util.cpp index 72da0f79865d..fdce932fd6d6 100644 --- a/openmp/runtime/src/z_Linux_util.cpp +++ b/openmp/runtime/src/z_Linux_util.cpp @@ -72,7 +72,7 @@ struct kmp_sys_timer { struct timespec start; }; -#if KMP_OS_SOLARIS +#ifndef TIMEVAL_TO_TIMESPEC // Convert timeval to timespec. #define TIMEVAL_TO_TIMESPEC(tv, ts) \ do { \ diff --git a/polly/lib/Analysis/ScopBuilder.cpp b/polly/lib/Analysis/ScopBuilder.cpp index c34413812d94..0af0f6915b14 100644 --- a/polly/lib/Analysis/ScopBuilder.cpp +++ b/polly/lib/Analysis/ScopBuilder.cpp @@ -2510,6 +2510,48 @@ static MemoryAccess::ReductionType getReductionType(const BinaryOperator *BinOp, } } +/// True if @p AllAccs intersects with @p MemAccs execpt @p LoadMA and @p +/// StoreMA +bool hasIntersectingAccesses(isl::set AllAccs, MemoryAccess *LoadMA, + MemoryAccess *StoreMA, isl::set Domain, + SmallVector &MemAccs) { + bool HasIntersectingAccs = false; + for (MemoryAccess *MA : MemAccs) { + if (MA == LoadMA || MA == StoreMA) + continue; + + isl::map AccRel = MA->getAccessRelation().intersect_domain(Domain); + isl::set Accs = AccRel.range(); + + if (AllAccs.has_equal_space(Accs)) { + isl::set OverlapAccs = Accs.intersect(AllAccs); + bool DoesIntersect = !OverlapAccs.is_empty(); + HasIntersectingAccs |= DoesIntersect; + } + } + return HasIntersectingAccs; +} + +/// Test if the accesses of @p LoadMA and @p StoreMA can form a reduction +bool checkCandidatePairAccesses(MemoryAccess *LoadMA, MemoryAccess *StoreMA, + isl::set Domain, + SmallVector &MemAccs) { + isl::map LoadAccs = LoadMA->getAccessRelation(); + isl::map StoreAccs = StoreMA->getAccessRelation(); + + // Skip those with obviously unequal base addresses. + bool Valid = LoadAccs.has_equal_space(StoreAccs); + + // And check if the remaining for overlap with other memory accesses. + if (Valid) { + isl::map AllAccsRel = LoadAccs.unite(StoreAccs); + AllAccsRel = AllAccsRel.intersect_domain(Domain); + isl::set AllAccs = AllAccsRel.range(); + Valid = !hasIntersectingAccesses(AllAccs, LoadMA, StoreMA, Domain, MemAccs); + } + return Valid; +} + void ScopBuilder::checkForReductions(ScopStmt &Stmt) { SmallVector Loads; SmallVector, 4> Candidates; @@ -2528,34 +2570,10 @@ void ScopBuilder::checkForReductions(ScopStmt &Stmt) { // Then check each possible candidate pair. for (const auto &CandidatePair : Candidates) { - bool Valid = true; - isl::map LoadAccs = CandidatePair.first->getAccessRelation(); - isl::map StoreAccs = CandidatePair.second->getAccessRelation(); - - // Skip those with obviously unequal base addresses. - if (!LoadAccs.has_equal_space(StoreAccs)) { - continue; - } - - // And check if the remaining for overlap with other memory accesses. - isl::map AllAccsRel = LoadAccs.unite(StoreAccs); - AllAccsRel = AllAccsRel.intersect_domain(Stmt.getDomain()); - isl::set AllAccs = AllAccsRel.range(); - - for (MemoryAccess *MA : Stmt) { - if (MA == CandidatePair.first || MA == CandidatePair.second) - continue; - - isl::map AccRel = - MA->getAccessRelation().intersect_domain(Stmt.getDomain()); - isl::set Accs = AccRel.range(); - - if (AllAccs.has_equal_space(Accs)) { - isl::set OverlapAccs = Accs.intersect(AllAccs); - Valid = Valid && OverlapAccs.is_empty(); - } - } - + MemoryAccess *LoadMA = CandidatePair.first; + MemoryAccess *StoreMA = CandidatePair.second; + bool Valid = checkCandidatePairAccesses(LoadMA, StoreMA, Stmt.getDomain(), + Stmt.MemAccs); if (!Valid) continue; @@ -2566,8 +2584,8 @@ void ScopBuilder::checkForReductions(ScopStmt &Stmt) { // If no overlapping access was found we mark the load and store as // reduction like. - CandidatePair.first->markAsReductionLike(RT); - CandidatePair.second->markAsReductionLike(RT); + LoadMA->markAsReductionLike(RT); + StoreMA->markAsReductionLike(RT); } } diff --git a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel index d3c7443748d9..477e59e366d1 100644 --- a/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/BUILD.bazel @@ -1934,10 +1934,12 @@ llvm_target_lib_list = [lib for lib in [ "name": "BPF", "short_name": "BPF", "tbl_outs": [ + ("-gen-register-bank", "lib/Target/BPF/BPFGenRegisterBank.inc"), ("-gen-asm-writer", "lib/Target/BPF/BPFGenAsmWriter.inc"), ("-gen-asm-matcher", "lib/Target/BPF/BPFGenAsmMatcher.inc"), ("-gen-callingconv", "lib/Target/BPF/BPFGenCallingConv.inc"), ("-gen-dag-isel", "lib/Target/BPF/BPFGenDAGISel.inc"), + ("-gen-global-isel", "lib/Target/BPF/BPFGenGlobalISel.inc"), ("-gen-disassembler", "lib/Target/BPF/BPFGenDisassemblerTables.inc"), ("-gen-emitter", "lib/Target/BPF/BPFGenMCCodeEmitter.inc"), ("-gen-instr-info", "lib/Target/BPF/BPFGenInstrInfo.inc"), @@ -2237,10 +2239,10 @@ gentbl( name = "riscv_isel_target_gen", strip_include_prefix = "lib/Target/RISCV", tbl_outs = [ - ("-gen-global-isel", "lib/Target/RISCV/RISCVGenGlobalISel.inc"), - ("-gen-global-isel-combiner -combiners=RISCVO0PreLegalizerCombiner", "lib/Target/RISCV/RISCVGenO0PreLegalizeGICombiner.inc"), - ("-gen-global-isel-combiner -combiners=RISCVPostLegalizerCombiner", "lib/Target/RISCV/RISCVGenPostLegalizeGICombiner.inc"), - ("-gen-global-isel-combiner -combiners=RISCVPreLegalizerCombiner", "lib/Target/RISCV/RISCVGenPreLegalizeGICombiner.inc"), + ("-gen-global-isel", "lib/Target/RISCV/RISCVGenGlobalISel.inc"), + ("-gen-global-isel-combiner -combiners=RISCVO0PreLegalizerCombiner", "lib/Target/RISCV/RISCVGenO0PreLegalizeGICombiner.inc"), + ("-gen-global-isel-combiner -combiners=RISCVPostLegalizerCombiner", "lib/Target/RISCV/RISCVGenPostLegalizeGICombiner.inc"), + ("-gen-global-isel-combiner -combiners=RISCVPreLegalizerCombiner", "lib/Target/RISCV/RISCVGenPreLegalizeGICombiner.inc"), ], tblgen = ":llvm-tblgen", td_file = "lib/Target/RISCV/RISCVGISel.td", @@ -2341,10 +2343,10 @@ gentbl( deps = [ ":BinaryFormat", ":CodeGenTypes", + ":Core", ":DebugInfoCodeView", ":MC", ":MCDisassembler", - ":Core", ":Support", ":Target", ":config", @@ -2550,23 +2552,23 @@ cc_library( ":AggressiveInstCombine", ":Analysis", ":CodeGen", - ":common_transforms", - ":config", ":Core", ":Coroutines", ":HipStdPar", - ":InstCombine", - ":Instrumentation", ":IPO", ":IRPrinter", + ":InstCombine", + ":Instrumentation", ":MLPolicies", ":ObjCARC", - ":pass_registry_def", ":Scalar", ":Support", ":Target", ":TransformUtils", ":Vectorize", + ":common_transforms", + ":config", + ":pass_registry_def", ], ) @@ -2802,20 +2804,19 @@ cc_library( ]), hdrs = glob([ "include/llvm/ExecutionEngine/Orc/Debugging/*.h", - ]), + ]) + ["include/llvm-c/LLJITUtils.h"], copts = llvm_copts, deps = [ ":BinaryFormat", ":DebugInfo", - ":OrcShared", - ":OrcJIT", ":JITLink", + ":OrcJIT", + ":OrcShared", ":Support", ":TargetParser", ], ) - cc_library( name = "OrcTargetProcess", srcs = glob([ @@ -3117,9 +3118,9 @@ cc_library( ":MC", ":MCA", ":MCDisassembler", - ":OrcJIT", ":Object", ":ObjectYAML", + ":OrcJIT", ":Support", ":Target", ":config", @@ -3249,9 +3250,9 @@ cc_binary( ":Interpreter", ":MCJIT", ":Object", + ":OrcDebugging", ":OrcJIT", ":OrcTargetProcess", - ":OrcDebugging", ":Support", ":TargetParser", ":config", @@ -4703,7 +4704,7 @@ cc_binary( srcs = glob([ "tools/llvm-symbolizer/*.cpp", "tools/llvm-symbolizer/*.h", - ]) + [ "llvm-symbolizer-driver.cpp"], + ]) + ["llvm-symbolizer-driver.cpp"], copts = llvm_copts, stamp = 0, deps = [ @@ -4816,7 +4817,6 @@ expand_template( template = "cmake/modules/llvm-driver-template.cpp.in", ) - cc_binary( name = "sancov", srcs = glob([ diff --git a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel index 42b417548d94..d2bcfc14ff75 100644 --- a/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/llvm/unittests/BUILD.bazel @@ -309,6 +309,7 @@ cc_test( "//llvm:JITLink", "//llvm:MC", "//llvm:Object", + "//llvm:OrcDebugging", "//llvm:OrcJIT", "//llvm:OrcShared", "//llvm:OrcTargetProcess", diff --git a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel index 7b64a4956e1e..c831df20349e 100644 --- a/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/BUILD.bazel @@ -3994,7 +3994,6 @@ cc_library( deps = [ ":ArithDialect", ":ArithUtils", - ":BufferizationDialect", ":ControlFlowDialect", ":ControlFlowInterfaces", ":DestinationStyleOpInterface", diff --git a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel index c4881383b5c5..d20bd403e8f8 100644 --- a/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel +++ b/utils/bazel/llvm-project-overlay/mlir/unittests/BUILD.bazel @@ -101,6 +101,7 @@ cc_test( "//llvm:Support", "//llvm:TestingSupport", "//mlir:Analysis", + "//mlir:Debug", "//mlir:FuncDialect", "//mlir:IR", "//mlir:Pass", -- GitLab

().g()), ""); @@ -214,57 +258,64 @@ namespace dr1330 { // dr1330: 4 c++11 static_assert(!noexcept(B().g()), ""); #endif - template int f() throw(typename T::error) { return 0; } // expected-error 1-4{{prior to '::'}} expected-note 0-1{{prior to '::'}} expected-note 0-1{{requested here}} -#if __cplusplus > 201402L - // expected-error@-2 0-1{{C++17}} expected-note@-2 0-1{{noexcept}} -#endif + template int f() throw(typename T::error) { return 0; } // #dr1330-f + // expected-error@#dr1330-f {{type 'int' cannot be used prior to '::' because it has no members}} + // cxx98-note@#dr1330-f-int {{in instantiation of function template specialization 'dr1330::f' requested here}} + // since-cxx11-note@#dr1330-f-int {{in instantiation of exception specification for 'f' requested here}} + // cxx98-14-error@#dr1330-f {{type 'short' cannot be used prior to '::' because it has no members}} + // cxx98-14-note@#dr1330-f-short {{in instantiation of function template specialization 'dr1330::f' requested here}} + // cxx11-14-note@#dr1330-f {{in instantiation of exception specification for 'f' requested here}} + // since-cxx11-error@#dr1330-f {{type 'char' cannot be used prior to '::' because it has no members}} + // since-cxx11-note@#dr1330-f-char {{in instantiation of exception specification for 'f' requested here}} + // since-cxx11-error@#dr1330-f {{type 'float' cannot be used prior to '::' because it has no members}} + // since-cxx11-note@#dr1330-f-float {{in instantiation of exception specification for 'f' requested here}} + // since-cxx17-error@#dr1330-f {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@#dr1330-f {{use 'noexcept(false)' instead}} + // An exception-specification is needed even if the function is only used in // an unevaluated operand. - int f1 = sizeof(f()); // expected-note {{instantiation of}} + int f1 = sizeof(f()); // #dr1330-f-int #if __cplusplus >= 201103L - decltype(f()) f2; // expected-note {{instantiation of}} - bool f3 = noexcept(f()); // expected-note {{instantiation of}} + decltype(f()) f2; // #dr1330-f-char + bool f3 = noexcept(f()); /// #dr1330-f-float #endif // In C++17 onwards, substituting explicit template arguments into the // function type substitutes into the exception specification (because it's // part of the type). In earlier languages, we don't notice there's a problem // until we've already started to instantiate. - template int f(); -#if __cplusplus >= 201703L - // expected-error@-2 {{does not refer to a function template}} -#else - // expected-note@-4 {{instantiation of}} -#endif + template int f(); // #dr1330-f-short + // since-cxx17-error@-1 {{explicit instantiation of 'f' does not refer to a function template, variable template, member function, member class, or static data member}} + // since-cxx17-note@#dr1330-f {{candidate template ignored: substitution failure [with T = short]: type 'short' cannot be used prior to '::' because it has no members}} template struct C { - C() throw(typename T::type); // expected-error 1-2{{prior to '::'}} -#if __cplusplus > 201402L - // expected-error@-2 0-1{{C++17}} expected-note@-2 0-1{{noexcept}} -#endif + C() throw(typename T::type); // #dr1330-C + // since-cxx17-error@-1 {{ISO C++17 does not allow dynamic exception specifications}} + // since-cxx17-note@-2 {{use 'noexcept(false)' instead}} + // cxx98-error@#dr1330-C {{type 'void' cannot be used prior to '::' because it has no members}} + // cxx98-note@#dr1330-C-void {{in instantiation of template class 'dr1330::C' requested here}} + // expected-error@#dr1330-C {{type 'int' cannot be used prior to '::' because it has no members}} + // cxx98-note@#dr1330-C-int {{in instantiation of template class 'dr1330::C' requested here}} + // since-cxx11-note@#dr1330-C-int {{in instantiation of exception specification for 'C' requested here}} + // since-cxx11-note@#dr1330-e {{in evaluation of exception specification for 'dr1330::E::E' needed here}} }; - struct D : C {}; // ok -#if __cplusplus < 201103L - // expected-note@-2 {{instantiation of}} -#endif + struct D : C {}; // #dr1330-C-void void f(D &d) { d = d; } // ok - struct E : C {}; // expected-note {{in instantiation of}} -#if __cplusplus >= 201103L - E e; // expected-note {{needed here}} -#endif + struct E : C {}; // #dr1330-C-int + E e; // #dr1330-e } namespace dr1341 { // dr1341: sup P0683R1 #if __cplusplus >= 202002L int a; -const int b = 0; // #dr1341-b-decl +const int b = 0; // #dr1341-b struct S { int x1 : 8 = 42; int x2 : 8 { 42 }; int y1 : true ? 8 : a = 42; int y2 : true ? 8 : b = 42; - // expected-error@-1 {{cannot assign to variable 'b' with const-qualified type 'const int'}} - // expected-note@#dr1341-b-decl {{variable 'b' declared const here}} + // since-cxx20-error@-1 {{cannot assign to variable 'b' with const-qualified type 'const int'}} + // since-cxx20-note@#dr1341-b {{variable 'b' declared const here}} int y3 : (true ? 8 : b) = 42; int z : 1 || new int { 0 }; }; @@ -272,44 +323,66 @@ struct S { } namespace dr1346 { // dr1346: 3.5 - auto a(1); // expected-error 0-1{{extension}} - auto b(1, 2); // expected-error {{multiple expressions}} expected-error 0-1{{extension}} + auto a(1); + // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}} + auto b(1, 2); + // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}} + // expected-error@-2 {{initializer for variable 'b' with type 'auto' contains multiple expressions}} #if __cplusplus >= 201103L - auto c({}); // expected-error {{parenthesized initializer list}} - auto d({1}); // expected-error {{parenthesized initializer list}} - auto e({1, 2}); // expected-error {{parenthesized initializer list}} + auto c({}); + // since-cxx11-error@-1 {{cannot deduce type for variable 'c' with type 'auto' from parenthesized initializer list}} + auto d({1}); + // since-cxx11-error@-1 {{cannot deduce type for variable 'd' with type 'auto' from parenthesized initializer list}} + auto e({1, 2}); + // since-cxx11-error@-1 {{cannot deduce type for variable 'e' with type 'auto' from parenthesized initializer list}} #endif - template void f(Ts ...ts) { // expected-error 0-1{{extension}} - auto x(ts...); // expected-error {{empty}} expected-error 0-1{{extension}} + template void f(Ts ...ts) { + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + auto x(ts...); + // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}} + // expected-error@-2 {{initializer for variable 'x' with type 'auto' is empty}} + // expected-note@#dr1346-f {{in instantiation of function template specialization 'dr1346::f<>' requested here}} } - template void f(); // expected-note {{instantiation}} + template void f(); // #dr1346-f #if __cplusplus >= 201103L void init_capture() { - [a(1)] {} (); // expected-error 0-1{{extension}} - [b(1, 2)] {} (); // expected-error {{multiple expressions}} expected-error 0-1{{extension}} -#if __cplusplus >= 201103L - [c({})] {} (); // expected-error {{parenthesized initializer list}} expected-error 0-1{{extension}} - [d({1})] {} (); // expected-error {{parenthesized initializer list}} expected-error 0-1{{extension}} - [e({1, 2})] {} (); // expected-error {{parenthesized initializer list}} expected-error 0-1{{extension}} -#endif + [a(1)] {} (); + // cxx11-error@-1 {{initialized lambda captures are a C++14 extension}} + [b(1, 2)] {} (); + // cxx11-error@-1 {{initialized lambda captures are a C++14 extension}} + // since-cxx11-error@-2 {{initializer for lambda capture 'b' contains multiple expressions}} + [c({})] {} (); + // cxx11-error@-1 {{initialized lambda captures are a C++14 extension}} + // since-cxx11-error@-2 {{cannot deduce type for lambda capture 'c' from parenthesized initializer list}} + [d({1})] {} (); + // cxx11-error@-1 {{initialized lambda captures are a C++14 extension}} + // since-cxx11-error@-2 {{cannot deduce type for lambda capture 'd' from parenthesized initializer list}} + [e({1, 2})] {} (); + // cxx11-error@-1 {{initialized lambda captures are a C++14 extension}} + // since-cxx11-error@-2 {{cannot deduce type for lambda capture 'e' from parenthesized initializer list}} } #endif } -namespace dr1347 { // dr1347: yes - auto x = 5, *y = &x; // expected-error 0-1{{extension}} - auto z = y, *q = y; // expected-error {{'auto' deduced as 'int *' in declaration of 'z' and deduced as 'int' in declaration of 'q'}} expected-error 0-1{{extension}} +namespace dr1347 { // dr1347: 3.1 + auto x = 5, *y = &x; + // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}} + auto z = y, *q = y; + // cxx98-error@-1 {{'auto' type specifier is a C++11 extension}} + // expected-error@-2 {{'auto' deduced as 'int *' in declaration of 'z' and deduced as 'int' in declaration of 'q'}} #if __cplusplus >= 201103L - auto a = 5, b = {1, 2}; // expected-error {{'auto' deduced as 'int' in declaration of 'a' and deduced as 'std::initializer_list' in declaration of 'b'}} - auto (*fp)(int) -> int, i = 0; // expected-error {{declaration with trailing return type must be the only declaration in its group}} + auto a = 5, b = {1, 2}; + // since-cxx11-error@-1 {{'auto' deduced as 'int' in declaration of 'a' and deduced as 'std::initializer_list' in declaration of 'b'}} + auto (*fp)(int) -> int, i = 0; + // since-cxx11-error@-1 {{declaration with trailing return type must be the only declaration in its group}} #endif } -namespace dr1358 { // dr1358: yes +namespace dr1358 { // dr1358: 3.1 #if __cplusplus >= 201103L struct Lit { constexpr operator int() const { return 0; } }; - struct NonLit { NonLit(); operator int(); }; // expected-note 2{{no constexpr constructors}} + struct NonLit { NonLit(); operator int(); }; // #dr1358-NonLit struct NonConstexprConv { constexpr operator int() const; }; struct Virt { virtual int f(int) const; }; @@ -336,8 +409,12 @@ namespace dr1358 { // dr1358: yes // But the corresponding non-template cases are rejected. struct B : Virt { int member; - constexpr B(NonLit u) : member(u) {} // expected-error {{not a literal type}} - constexpr NonLit f(NonLit u) const { return NonLit(); } // expected-error {{not a literal type}} + constexpr B(NonLit u) : member(u) {} + // since-cxx11-error@-1 {{constexpr constructor's 1st parameter type 'NonLit' is not a literal type}} + // since-cxx11-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} + constexpr NonLit f(NonLit u) const { return NonLit(); } + // since-cxx11-error@-1 {{constexpr function's return type 'NonLit' is not a literal type}} + // since-cxx11-note@#dr1358-NonLit {{'NonLit' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} }; #endif } @@ -345,65 +422,113 @@ namespace dr1358 { // dr1358: yes namespace dr1359 { // dr1359: 3.5 #if __cplusplus >= 201103L union A { constexpr A() = default; }; - union B { constexpr B() = default; int a; }; // expected-error {{not constexpr}} expected-note 2{{candidate}} - union C { constexpr C() = default; int a, b; }; // expected-error {{not constexpr}} expected-note 2{{candidate}} - struct X { constexpr X() = default; union {}; }; // expected-error {{does not declare anything}} - struct Y { constexpr Y() = default; union { int a; }; }; // expected-error {{not constexpr}} expected-note 2{{candidate}} + union B { constexpr B() = default; int a; }; // #dr1359-B + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} + union C { constexpr C() = default; int a, b; }; // #dr1359-C + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} + struct X { constexpr X() = default; union {}; }; + // since-cxx11-error@-1 {{declaration does not declare anything}} + struct Y { constexpr Y() = default; union { int a; }; }; // #dr1359-Y + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} constexpr A a = A(); - constexpr B b = B(); // expected-error {{no matching}} - constexpr C c = C(); // expected-error {{no matching}} + constexpr B b = B(); + // cxx11-17-error@-1 {{no matching constructor for initialization of 'B'}} + // cxx11-17-note@#dr1359-B {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}} + // cxx11-17-note@#dr1359-B {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}} + constexpr C c = C(); + // cxx11-17-error@-1 {{no matching constructor for initialization of 'C'}} + // cxx11-17-note@#dr1359-C {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}} + // cxx11-17-note@#dr1359-C {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}} constexpr X x = X(); - constexpr Y y = Y(); // expected-error {{no matching}} + constexpr Y y = Y(); + // cxx11-17-error@-1 {{no matching constructor for initialization of 'Y'}} + // cxx11-17-note@#dr1359-Y {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}} + // cxx11-17-note@#dr1359-Y {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}} #endif } namespace dr1388 { // dr1388: 4 - template void f(T..., A); // expected-note 1+{{candidate}} expected-error 0-1{{C++11}} - template void g(T..., int); // expected-note 1+{{candidate}} expected-error 0-1{{C++11}} - template void h(T..., A); // expected-note 1+{{candidate}} expected-error 0-1{{C++11}} + template void f(T..., A); // #dr1388-f + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + template void g(T..., int); // #dr1388-g + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + template void h(T..., A); // #dr1388-h + // cxx98-error@-1 {{variadic templates are a C++11 extension}} void test_f() { f(0); // ok, trailing parameter pack deduced to empty - f(0, 0); // expected-error {{no matching}} + f(0, 0); + // expected-error@-1 {{no matching function for call to 'f'}} + // expected-note@#dr1388-f {{candidate function [with A = int, T = <>] not viable: requires 1 argument, but 2 were provided}} f(0); - f(0, 0); // expected-error {{no matching}} + f(0, 0); + // expected-error@-1 {{no matching function for call to 'f'}} + // expected-note@#dr1388-f {{candidate function [with A = int, T = <>] not viable: requires 1 argument, but 2 were provided}} f(0, 0); - f(0, 0); // expected-error {{no matching}} + f(0, 0); + // expected-error@-1 {{no matching function for call to 'f'}} + // expected-note@#dr1388-f {{candidate function [with A = int, T = ] not viable: requires 3 arguments, but 2 were provided}} g(0); - g(0, 0); // expected-error {{no matching}} + g(0, 0); + // expected-error@-1 {{no matching function for call to 'g'}} + // expected-note@#dr1388-g {{candidate function [with T = <>] not viable: requires 1 argument, but 2 were provided}} g<>(0); - g(0); // expected-error {{no matching}} + g(0); + // expected-error@-1 {{no matching function for call to 'g'}} + // expected-note@#dr1388-g {{candidate function [with T = ] not viable: requires 2 arguments, but 1 was provided}} g(0, 0); h(0); - h(0, 0); // expected-error {{no matching}} + h(0, 0); + // expected-error@-1 {{no matching function for call to 'h'}} + // expected-note@#dr1388-h {{candidate function [with T = <>, A = int] not viable: requires 1 argument, but 2 were provided}} h(0, 0); - h(0, 0); // expected-error {{no matching}} + h(0, 0); + // expected-error@-1 {{no matching function for call to 'h'}} + // expected-note@#dr1388-h {{candidate template ignored: couldn't infer template argument 'A'}} } // A non-trailing parameter pack is still a non-deduced context, even though // we know exactly how many arguments correspond to it. template struct pair {}; - template struct tuple { typedef char type; }; // expected-error 0-2{{C++11}} - template void f_pair_1(pair..., int); // expected-error 0-2{{C++11}} expected-note {{[with T = ]: deduced incomplete pack <(no value), (no value)> for template parameter 'U'}} - template void f_pair_2(pair..., U); // expected-error 0-2{{C++11}} - template void f_pair_3(pair..., tuple); // expected-error 0-2{{C++11}} expected-note {{deduced packs of different lengths for parameter 'U' (<(no value), (no value)> vs. )}} - template void f_pair_4(pair..., T...); // expected-error 0-2{{C++11}} expected-note {{ vs. }} + template struct tuple { typedef char type; }; // + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + template void f_pair_1(pair..., int); // #dr1388-f-1 + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + // cxx98-error@-2 {{variadic templates are a C++11 extension}} + template void f_pair_2(pair..., U); + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + template void f_pair_3(pair..., tuple); // #dr1388-f-3 + // cxx98-error@-1 {{variadic templates are a C++11 extension}} + // cxx98-error@-2 {{variadic templates are a C++11 extension}} + template void f_pair_4(pair..., T...); // #dr1388-f-4 + // cxx98-error@-1 {{variadic templates are a C++11 extension}} void g(pair a, pair b, tuple c) { - f_pair_1(a, b, 0); // expected-error {{no match}} + f_pair_1(a, b, 0); + // expected-error@-1 {{no matching function for call to 'f_pair_1'}} + // expected-note@#dr1388-f-1 {{candidate template ignored: substitution failure [with T = ]: deduced incomplete pack <(no value), (no value)> for template parameter 'U'}} f_pair_2(a, b, 0); f_pair_3(a, b, c); - f_pair_3(a, b, tuple()); // expected-error {{no match}} + f_pair_3(a, b, tuple()); + // expected-error@-1 {{no matching function for call to 'f_pair_3'}} + // expected-note@#dr1388-f-3 {{candidate template ignored: deduced packs of different lengths for parameter 'U' (<(no value), (no value)> vs. )}} f_pair_4(a, b, 0, 0L); - f_pair_4(a, b, 0, 0L, "foo"); // expected-error {{no match}} + f_pair_4(a, b, 0, 0L, "foo"); + // expected-error@-1 {{no matching function for call to 'f_pair_4'}} + // expected-note@#dr1388-f-4 {{candidate template ignored: deduced packs of different lengths for parameter 'T' ( vs. )}} } } namespace dr1391 { // dr1391: partial struct A {}; struct B : A {}; - template struct C { C(int); typename T::error error; }; // expected-error 2{{'::'}} + template struct C { C(int); typename T::error error; }; // #dr1391-C + // expected-error@#dr1391-C {{type 'int' cannot be used prior to '::' because it has no members}} + // expected-note@#dr1391-b {{in instantiation of template class 'dr1391::C' requested here}} + // expected-note@#dr1391-b {{while substituting deduced template arguments into function template 'b' [with T = int]}} + // expected-error@#dr1391-C {{type 'double' cannot be used prior to '::' because it has no members}} + // expected-note@#dr1391-c {{in instantiation of template class 'dr1391::C' requested here}} template struct D {}; // No deduction is performed for parameters with no deducible template-parameters, therefore types do not need to match. @@ -444,28 +569,36 @@ namespace dr1391 { // dr1391: partial void test_b() { b(0, 0); // ok, deduction fails prior to forming a conversion sequence and instantiating C // FIXME: The "while substituting" note should point at the overload candidate. - b(0, 0); // expected-note {{instantiation of}} expected-note {{while substituting}} + b(0, 0); // #dr1391-b } template struct Id { typedef T type; }; template void c(T, typename Id >::type); void test_c() { // Implicit conversion sequences for dependent types are checked later. - c(0.0, 0); // expected-note {{instantiation of}} + c(0.0, 0); // #dr1391-c } namespace partial_ordering { // FIXME: Second template should be considered more specialized because non-dependent parameter is ignored. - template int a(T, short) = delete; // expected-error 0-1{{extension}} expected-note {{candidate}} - template int a(T*, char); // expected-note {{candidate}} - int test_a = a((int*)0, 0); // FIXME: expected-error {{ambiguous}} + template int a(T, short) = delete; // #dr1391-a-short + // cxx98-error@-1 {{deleted function definitions are a C++11 extension}} + template int a(T*, char); // #dr1391-a-char + int test_a = a((int*)0, 0); + // expected-error@-1 {{call to 'a' is ambiguous}} FIXME + // expected-note@#dr1391-a-short {{candidate function [with T = int *] has been explicitly deleted}} + // expected-note@#dr1391-a-char {{candidate function [with T = int]}} // FIXME: Second template should be considered more specialized: // deducing #1 from #2 ignores the second P/A pair, so deduction succeeds, // deducing #2 from #1 fails to deduce T, so deduction fails. - template int b(T, int) = delete; // expected-error 0-1{{extension}} expected-note {{candidate}} - template int b(T*, U); // expected-note {{candidate}} - int test_b = b((int*)0, 0); // FIXME: expected-error {{ambiguous}} + template int b(T, int) = delete; // #dr1391-b-int + // cxx98-error@-1 {{deleted function definitions are a C++11 extension}} + template int b(T*, U); // #dr1391-b-U + int test_b = b((int*)0, 0); + // expected-error@-1 {{call to 'b' is ambiguous}} FIXME + // expected-note@#dr1391-b-int {{candidate function [with T = int *] has been explicitly deleted}} + // expected-note@#dr1391-b-U {{candidate function [with T = int, U = int]}} // Unintended consequences: because partial ordering does not consider // explicit template arguments, and deduction from a non-dependent type @@ -499,26 +632,26 @@ namespace dr1395 { // dr1395: 16 namespace dr1397 { // dr1397: 3.2 #if __cplusplus >= 201103L -struct A { // #dr1397-struct-A - void *p = A{}; // #dr1397-void-p -#if __cplusplus == 201103L - // expected-error@#dr1397-struct-A {{default member initializer for 'p' needed within definition of enclosing class 'A' outside of member functions}} - // expected-note@#dr1397-void-p {{in evaluation of exception specification for 'dr1397::A::A' needed here}} - // expected-note@#dr1397-void-p {{default member initializer declared here}} -#elif __cplusplus >= 201402L - // expected-error@#dr1397-void-p {{default member initializer for 'p' needed within definition of enclosing class 'A' outside of member functions}} - // expected-note@#dr1397-void-p {{default member initializer declared here}} -#endif +struct A { +// cxx11-error@-1 {{default member initializer for 'p' needed within definition of enclosing class 'A' outside of member functions}} +// cxx11-note@#dr1397-p {{in evaluation of exception specification for 'dr1397::A::A' needed here}} +// cxx11-note@#dr1397-p {{default member initializer declared here}} + void *p = A{}; // #dr1397-p + // since-cxx14-error@-1 {{default member initializer for 'p' needed within definition of enclosing class 'A' outside of member functions}} + // since-cxx14-note@-2 {{default member initializer declared here}} operator void*() const { return nullptr; } }; #endif } // namespace dr1397 namespace dr1399 { // dr1399: dup 1388 - template void f(T..., int, T...) {} // expected-note {{candidate}} expected-error 0-1{{C++11}} + template void f(T..., int, T...) {} // #dr1399-f + // cxx98-error@-1 {{variadic templates are a C++11 extension}} void g() { f(0); f(0, 0, 0); - f(0, 0, 0); // expected-error {{no match}} + f(0, 0, 0); + // expected-error@-1 {{no matching function for call to 'f'}} + // expected-note@#dr1399-f {{candidate template ignored: deduced packs of different lengths for parameter 'T' (<> vs. )}} } } diff --git a/clang/test/CXX/drs/dr14xx.cpp b/clang/test/CXX/drs/dr14xx.cpp index ea41a03d3587..d262f6f9dcab 100644 --- a/clang/test/CXX/drs/dr14xx.cpp +++ b/clang/test/CXX/drs/dr14xx.cpp @@ -1,8 +1,10 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2a %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx11-17,since-cxx11, -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,cxx14-17,cxx11-17,since-cxx11,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,cxx14-17,cxx11-17,since-cxx11,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors namespace dr1413 { // dr1413: 12 template struct Check { @@ -15,24 +17,41 @@ namespace dr1413 { // dr1413: 12 void d(); void f() { - Check::type *var1; // expected-error {{undeclared identifier 'var1'}} - Check::type *var2; // ok, variable declaration expected-note 0+{{here}} - Check::type *var3; // expected-error {{undeclared identifier 'var3'}} - Check::type *var4; // expected-error {{undeclared identifier 'var4'}} + Check::type *var1; + // expected-error@-1 {{use of undeclared identifier 'var1'}} + + // ok, variable declaration + Check::type *var2; // #dr1413-var2 + Check::type *var3; + // expected-error@-1 {{use of undeclared identifier 'var3'}} + // expected-note@#dr1413-var2 {{'var2' declared here}} + Check::type *var4; + // expected-error@-1 {{use of undeclared identifier 'var4'}} + // expected-note@#dr1413-var2 {{'var2' declared here}} + // value-dependent because of the implied type-dependent 'this->', not because of 'd' - Check::type *var5; // expected-error {{undeclared identifier 'var5'}} + Check::type *var5; + // expected-error@-1 {{use of undeclared identifier 'var5'}} + // expected-note@#dr1413-var2 {{'var2' declared here}} + // value-dependent because of the value-dependent '&' operator, not because of 'A::d' - Check::type *var5; // expected-error {{undeclared identifier 'var5'}} + Check::type *var5; + // expected-error@-1 {{use of undeclared identifier 'var5'}} + // expected-note@#dr1413-var2 {{'var2' declared here}} } }; } namespace dr1423 { // dr1423: 11 #if __cplusplus >= 201103L - bool b1 = nullptr; // expected-error {{cannot initialize}} - bool b2(nullptr); // expected-warning {{implicit conversion of nullptr constant to 'bool'}} - bool b3 = {nullptr}; // expected-error {{cannot initialize}} - bool b4{nullptr}; // expected-warning {{implicit conversion of nullptr constant to 'bool'}} + bool b1 = nullptr; + // since-cxx11-error@-1 {{cannot initialize a variable of type 'bool' with an rvalue of type 'std::nullptr_t'}} + bool b2(nullptr); + // since-cxx11-warning@-1 {{implicit conversion of nullptr constant to 'bool'}} + bool b3 = {nullptr}; + // since-cxx11-error@-1 {{cannot initialize a variable of type 'bool' with an rvalue of type 'std::nullptr_t'}} + bool b4{nullptr}; + // since-cxx11-warning@-1 {{implicit conversion of nullptr constant to 'bool'}} #endif } @@ -62,7 +81,8 @@ namespace dr1432 { // dr1432: 16 namespace dr1443 { // dr1443: yes struct A { int i; - A() { void foo(int=i); } // expected-error {{default argument references 'this'}} + A() { void foo(int=i); } + // expected-error@-1 {{default argument references 'this'}} }; } @@ -70,49 +90,54 @@ namespace dr1460 { // dr1460: 3.5 #if __cplusplus >= 201103L namespace DRExample { union A { - union {}; // expected-error {{does not declare anything}} - union {}; // expected-error {{does not declare anything}} + union {}; + // expected-error@-1 {{declaration does not declare anything}} + union {}; + // expected-error@-1 {{declaration does not declare anything}} constexpr A() {} }; constexpr A a = A(); union B { - union {}; // expected-error {{does not declare anything}} - union {}; // expected-error {{does not declare anything}} + union {}; + // expected-error@-1 {{declaration does not declare anything}} + union {}; + // expected-error@-1 {{declaration does not declare anything}} constexpr B() = default; }; constexpr B b = B(); union C { - union {}; // expected-error {{does not declare anything}} - union {}; // expected-error {{does not declare anything}} + union {}; + // expected-error@-1 {{declaration does not declare anything}} + union {}; + // expected-error@-1 {{declaration does not declare anything}} }; constexpr C c = C(); -#if __cplusplus > 201103L +#if __cplusplus >= 201403L constexpr void f() { C c; } static_assert((f(), true), ""); #endif } union A {}; - union B { int n; }; // expected-note 0+{{here}} + union B { int n; }; // #dr1460-B union C { int n = 0; }; - struct D { union {}; }; // expected-error {{does not declare anything}} - struct E { union { int n; }; }; // expected-note 0+{{here}} + struct D { union {}; }; + // expected-error@-1 {{declaration does not declare anything}} + struct E { union { int n; }; }; // #dr1460-E struct F { union { int n = 0; }; }; struct X { friend constexpr A::A() noexcept; friend constexpr B::B() noexcept; -#if __cplusplus <= 201703L - // expected-error@-2 {{follows non-constexpr declaration}} -#endif + // cxx11-17-error@-1 {{constexpr declaration of 'B' follows non-constexpr declaration}} + // cxx11-17-note@#dr1460-B {{previous declaration is here}} friend constexpr C::C() noexcept; friend constexpr D::D() noexcept; friend constexpr E::E() noexcept; -#if __cplusplus <= 201703L - // expected-error@-2 {{follows non-constexpr declaration}} -#endif + // cxx11-17-error@-1 {{constexpr declaration of 'E' follows non-constexpr declaration}} + // cxx11-17-note@#dr1460-E {{previous declaration is here}} friend constexpr F::F() noexcept; }; @@ -128,79 +153,77 @@ namespace dr1460 { // dr1460: 3.5 namespace Defaulted { union A { constexpr A() = default; }; union B { int n; constexpr B() = default; }; -#if __cplusplus <= 201703L - // expected-error@-2 {{not constexpr}} -#endif + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} union C { int n = 0; constexpr C() = default; }; - struct D { union {}; constexpr D() = default; }; // expected-error {{does not declare anything}} + struct D { union {}; constexpr D() = default; }; + // expected-error@-1 {{declaration does not declare anything}} struct E { union { int n; }; constexpr E() = default; }; -#if __cplusplus <= 201703L - // expected-error@-2 {{not constexpr}} -#endif + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} struct F { union { int n = 0; }; constexpr F() = default; }; struct G { union { int n = 0; }; union { int m; }; constexpr G() = default; }; -#if __cplusplus <= 201703L - // expected-error@-2 {{not constexpr}} -#endif + // cxx11-17-error@-1 {{defaulted definition of default constructor is not constexpr}} struct H { union { int n = 0; }; - union { // expected-note 0-2{{member not initialized}} + union { // #dr1460-H-union int m; }; constexpr H() {} -#if __cplusplus <= 201703L - // expected-error@-2 {{initialize all members}} -#endif + // cxx11-17-error@-1 {{constexpr constructor that does not initialize all members is a C++20 extension}} + // cxx11-17-note@#dr1460-H-union {{member not initialized by constructor}} constexpr H(bool) : m(1) {} constexpr H(char) : n(1) {} -#if __cplusplus <= 201703L - // expected-error@-2 {{initialize all members}} -#endif + // cxx11-17-error@-1 {{constexpr constructor that does not initialize all members is a C++20 extension}} + // cxx11-17-note@#dr1460-H-union {{member not initialized by constructor}} constexpr H(double) : m(1), n(1) {} }; } -#if __cplusplus > 201103L +#if __cplusplus >= 201403L template constexpr bool check() { - T t; -#if __cplusplus <= 201703L - // expected-note-re@-2 2{{non-constexpr constructor '{{[BE]}}'}} -#endif + T t; // #dr1460-t return true; } static_assert(check(), ""); - static_assert(check(), ""); -#if __cplusplus <= 201703L - // expected-error@-2 {{constant}} expected-note@-2 {{in call}} -#endif + static_assert(check(), ""); // #dr1460-check-B + // cxx14-17-error@-1 {{static assertion expression is not an integral constant expression}} + // cxx14-17-note@#dr1460-t {{non-constexpr constructor 'B' cannot be used in a constant expression}} + // cxx14-17-note@#dr1460-check-B {{in call to 'check()'}} + // cxx14-17-note@#dr1460-B {{declared here}} static_assert(check(), ""); static_assert(check(), ""); - static_assert(check(), ""); -#if __cplusplus <= 201703L - // expected-error@-2 {{constant}} expected-note@-2 {{in call}} -#endif + static_assert(check(), ""); // #dr1460-check-E + // cxx14-17-error@-1 {{static assertion expression is not an integral constant expression}} + // cxx14-17-note@#dr1460-t {{non-constexpr constructor 'E' cannot be used in a constant expression}} + // cxx14-17-note@#dr1460-check-E {{in call to 'check()'}} + // cxx14-17-note@#dr1460-E {{declared here}} static_assert(check(), ""); #endif union G { - int a = 0; // expected-note {{previous initialization is here}} - int b = 0; // expected-error {{initializing multiple members of union}} + int a = 0; // #dr1460-G-a + int b = 0; + // expected-error@-1 {{initializing multiple members of union}} + // expected-note@#dr1460-G-a {{previous initialization is here}} }; union H { union { - int a = 0; // expected-note {{previous initialization is here}} + int a = 0; // #dr1460-H-a }; union { - int b = 0; // expected-error {{initializing multiple members of union}} + int b = 0; + // expected-error@-1 {{initializing multiple members of union}} + // expected-note@#dr1460-H-a {{previous initialization is here}} }; }; struct I { union { - int a = 0; // expected-note {{previous initialization is here}} - int b = 0; // expected-error {{initializing multiple members of union}} + int a = 0; // #dr1460-I-a + int b = 0; + // expected-error@-1 {{initializing multiple members of union}} + // expected-note@#dr1460-I-a {{previous initialization is here}} }; }; struct J { @@ -223,14 +246,24 @@ namespace dr1460 { // dr1460: 3.5 constexpr B(const char*) {} }; static_assert(B().a == 1, ""); - static_assert(B().b == 2, ""); // expected-error {{constant}} expected-note {{read of}} - static_assert(B('x').a == 0, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(B().b == 2, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'b' of union with active member 'a' is not allowed in a constant expression}} + static_assert(B('x').a == 0, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'a' of union with active member 'b' is not allowed in a constant expression}} static_assert(B('x').b == 4, ""); - static_assert(B(123).b == 2, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(B(123).b == 2, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'b' of union with active member 'c' is not allowed in a constant expression}} static_assert(B(123).c == 3, ""); - static_assert(B("").a == 1, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(B("").a == 1, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'a' of union with active member 'b' is not allowed in a constant expression}} static_assert(B("").b == 2, ""); - static_assert(B("").c == 3, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(B("").c == 3, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'c' of union with active member 'b' is not allowed in a constant expression}} struct C { union { int a, b = 2, c; }; @@ -243,31 +276,55 @@ namespace dr1460 { // dr1460: 3.5 }; static_assert(C().a == 1, ""); - static_assert(C().b == 2, ""); // expected-error {{constant}} expected-note {{read of}} - static_assert(C().d == 4, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C().b == 2, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'b' of union with active member 'a' is not allowed in a constant expression}} + static_assert(C().d == 4, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'd' of union with active member 'e' is not allowed in a constant expression}} static_assert(C().e == 5, ""); - static_assert(C('x').b == 2, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C('x').b == 2, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'b' of union with active member 'c' is not allowed in a constant expression}} static_assert(C('x').c == 3, ""); - static_assert(C('x').d == 4, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C('x').d == 4, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'd' of union with active member 'e' is not allowed in a constant expression}} static_assert(C('x').e == 5, ""); static_assert(C(1).b == 2, ""); - static_assert(C(1).c == 3, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C(1).c == 3, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'c' of union with active member 'b' is not allowed in a constant expression}} static_assert(C(1).d == 4, ""); - static_assert(C(1).e == 5, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C(1).e == 5, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'e' of union with active member 'd' is not allowed in a constant expression}} static_assert(C(1.f).b == 2, ""); - static_assert(C(1.f).c == 3, ""); // expected-error {{constant}} expected-note {{read of}} - static_assert(C(1.f).e == 5, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C(1.f).c == 3, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'c' of union with active member 'b' is not allowed in a constant expression}} + static_assert(C(1.f).e == 5, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'e' of union with active member 'f' is not allowed in a constant expression}} static_assert(C(1.f).f == 6, ""); - static_assert(C("").a == 1, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C("").a == 1, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'a' of union with active member 'b' is not allowed in a constant expression}} static_assert(C("").b == 2, ""); - static_assert(C("").c == 3, ""); // expected-error {{constant}} expected-note {{read of}} - static_assert(C("").d == 4, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C("").c == 3, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'c' of union with active member 'b' is not allowed in a constant expression}} + static_assert(C("").d == 4, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'd' of union with active member 'e' is not allowed in a constant expression}} static_assert(C("").e == 5, ""); - static_assert(C("").f == 6, ""); // expected-error {{constant}} expected-note {{read of}} + static_assert(C("").f == 6, ""); + // expected-error@-1 {{static assertion expression is not an integral constant expression}} + // expected-note@-2 {{read of member 'f' of union with active member 'e' is not allowed in a constant expression}} struct D; extern const D d; @@ -315,8 +372,10 @@ namespace std { const _E* end() const {return __begin_ + __size_;} }; } // std +#endif namespace dr1467 { // dr1467: 3.7 c++11 +#if __cplusplus >= 201103L // Note that the change to [over.best.ics] was partially undone by DR2076; // the resulting rule is tested with the tests for that change. @@ -382,9 +441,12 @@ namespace dr1467 { // dr1467: 3.7 c++11 X x; X x2{x}; - void f1(int); // expected-note {{candidate function}} - void f1(std::initializer_list) = delete; // expected-note {{candidate function has been explicitly deleted}} - void g1() { f1({42}); } // expected-error {{call to deleted function 'f1'}} + void f1(int); // #dr1467-f1 + void f1(std::initializer_list) = delete; // #dr1467-f1-deleted + void g1() { f1({42}); } + // since-cxx11-error@-1 {{call to deleted function 'f1'}} + // since-cxx11-note@#dr1467-f1 {{candidate function}} + // since-cxx11-note@#dr1467-f1-deleted {{candidate function has been explicitly deleted}} template struct Pair { @@ -394,9 +456,12 @@ namespace dr1467 { // dr1467: 3.7 c++11 String(const char *); }; - void f2(Pair); // expected-note {{candidate function}} - void f2(std::initializer_list) = delete; // expected-note {{candidate function has been explicitly deleted}} - void g2() { f2({"foo", "bar"}); } // expected-error {{call to deleted function 'f2'}} + void f2(Pair); // #dr1467-f2 + void f2(std::initializer_list) = delete; // #dr1467-f2-deleted + void g2() { f2({"foo", "bar"}); } + // since-cxx11-error@-1 {{call to deleted function 'f2'}} + // since-cxx11-note@#dr1467-f2 {{candidate function}} + // since-cxx11-note@#dr1467-f2-deleted {{candidate function has been explicitly deleted}} } // dr_example namespace nonaggregate { @@ -453,84 +518,151 @@ namespace dr1467 { // dr1467: 3.7 c++11 } } // namespace NonAmbiguous -#if __cplusplus >= 201103L namespace StringLiterals { // When the array size is 4 the call will attempt to bind an lvalue to an // rvalue and fail. Therefore #2 will be called. (rsmith will bring this // issue to CWG) - void f(const char(&&)[4]); // expected-note 2 {{expects an rvalue}} expected-note 3 {{no known conversion}} - void f(const char(&&)[5]) = delete; // expected-note 2 {{candidate function has been explicitly deleted}} expected-note 3 {{no known conversion}} - void f(const wchar_t(&&)[4]); // expected-note {{expects an rvalue}} expected-note 4 {{no known conversion}} - void f(const wchar_t(&&)[5]) = delete; // expected-note {{candidate function has been explicitly deleted}} expected-note 4 {{no known conversion}} + void f(const char(&&)[4]); // #dr1467-f-char-4 + void f(const char(&&)[5]) = delete; // #dr1467-f-char-5 + void f(const wchar_t(&&)[4]); // #dr1467-f-wchar-4 + void f(const wchar_t(&&)[5]) = delete; // #dr1467-f-wchar-5 #if __cplusplus >= 202002L - void f2(const char8_t(&&)[4]); // expected-note {{expects an rvalue}} - void f2(const char8_t(&&)[5]) = delete; // expected-note {{candidate function has been explicitly deleted}} + void f2(const char8_t(&&)[4]); // #dr1467-f2-char8-4 + void f2(const char8_t(&&)[5]) = delete; // #dr1467-f2-char8-5 #endif - void f(const char16_t(&&)[4]); // expected-note {{expects an rvalue}} expected-note 4 {{no known conversion}} - void f(const char16_t(&&)[5]) = delete; // expected-note {{candidate function has been explicitly deleted}} expected-note 4 {{no known conversion}} - void f(const char32_t(&&)[4]); // expected-note {{expects an rvalue}} expected-note 4 {{no known conversion}} - void f(const char32_t(&&)[5]) = delete; // expected-note {{candidate function has been explicitly deleted}} expected-note 4 {{no known conversion}} + void f(const char16_t(&&)[4]); // #dr1467-f-char16-4 + void f(const char16_t(&&)[5]) = delete; // #dr1467-f-char16-5 + void f(const char32_t(&&)[4]); // #dr1467-f-char32-4 + void f(const char32_t(&&)[5]) = delete; // #dr1467-f-char32-5 void g() { - f({"abc"}); // expected-error {{call to deleted function 'f'}} - f({((("abc")))}); // expected-error {{call to deleted function 'f'}} - f({L"abc"}); // expected-error {{call to deleted function 'f'}} + f({"abc"}); + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr1467-f-char-5 {{candidate function has been explicitly deleted}} + // since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: expects an rvalue for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}} + f({((("abc")))}); + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr1467-f-char-5 {{candidate function has been explicitly deleted}} + // since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: expects an rvalue for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-5 {{candidate function not viable: no known conversion from 'const char[4]' to 'const char32_t' for 1st argument}} + f({L"abc"}); + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr1467-f-wchar-5 {{candidate function has been explicitly deleted}} + // since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char' for 1st argument}} + // since-cxx11-note@#dr1467-f-char-5 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: expects an rvalue for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-5 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char32_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-5 {{candidate function not viable: no known conversion from 'const wchar_t[4]' to 'const char32_t' for 1st argument}} #if __cplusplus >= 202002L - f2({u8"abc"}); // expected-error {{call to deleted function 'f2'}} + f2({u8"abc"}); + // since-cxx20-error@-1 {{call to deleted function 'f2'}} + // since-cxx20-note@#dr1467-f2-char8-5 {{candidate function has been explicitly deleted}} + // since-cxx20-note@#dr1467-f2-char8-4 {{candidate function not viable: expects an rvalue for 1st argument}} #endif - f({uR"(abc)"}); // expected-error {{call to deleted function 'f'}} - f({(UR"(abc)")}); // expected-error {{call to deleted function 'f'}} + f({uR"(abc)"}); + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr1467-f-char16-5 {{candidate function has been explicitly deleted}} + // since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char' for 1st argument}} + // since-cxx11-note@#dr1467-f-char-5 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: expects an rvalue for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char32_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-5 {{candidate function not viable: no known conversion from 'const char16_t[4]' to 'const char32_t' for 1st argument}} + f({(UR"(abc)")}); + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr1467-f-char32-5 {{candidate function has been explicitly deleted}} + // since-cxx11-note@#dr1467-f-char-4 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char' for 1st argument}} + // since-cxx11-note@#dr1467-f-char-5 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-4 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-wchar-5 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const wchar_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-4 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char16-5 {{candidate function not viable: no known conversion from 'const char32_t[4]' to 'const char16_t' for 1st argument}} + // since-cxx11-note@#dr1467-f-char32-4 {{candidate function not viable: expects an rvalue for 1st argument}} } } // namespace StringLiterals #endif } // dr1467 -namespace dr1479 { // dr1479: yes - int operator"" _a(const char*, std::size_t = 0); // expected-error {{literal operator cannot have a default argument}} +namespace dr1479 { // dr1479: 3.1 +#if __cplusplus >= 201103L + int operator"" _a(const char*, std::size_t = 0); + // since-cxx11-error@-1 {{literal operator cannot have a default argument}} +#endif } -namespace dr1482 { // dr1482: yes +namespace dr1482 { // dr1482: 3.0 // NB: sup 2516, test reused there #if __cplusplus >= 201103L template struct S { typedef char I; }; enum E2 : S::I { e }; -// expected-error@-1 {{use of undeclared identifier 'E2'}} +// since-cxx11-error@-1 {{use of undeclared identifier 'E2'}} #endif } // namespace dr1482 namespace dr1490 { // dr1490: 3.7 c++11 +#if __cplusplus >= 201103L // List-initialization from a string literal - char s[4]{"abc"}; // Ok - std::initializer_list{"abc"}; // expected-error {{expected unqualified-id}}} + char s[4]{"abc"}; // Ok + std::initializer_list{"abc"}; + // since-cxx11-error@-1 {{expected unqualified-id}}} +#endif } // dr1490 namespace dr1495 { // dr1495: 4 +#if __cplusplus >= 201103L // Deduction succeeds in both directions. - template struct A {}; // expected-note {{template is declared here}} - template struct A {}; // expected-error {{class template partial specialization is not more specialized}} + template struct A {}; // #dr1495-A + template struct A {}; + // since-cxx11-error@-1 {{class template partial specialization is not more specialized than the primary template}} + // since-cxx11-note@#dr1495-A {{template is declared here}} // Primary template is more specialized. - template struct B {}; // expected-note {{template is declared here}} - template struct B {}; // expected-error {{not more specialized}} + template struct B {}; // #dr1495-B + template struct B {}; + // since-cxx11-error@-1 {{class template partial specialization is not more specialized than the primary template}} + // since-cxx11-note@#dr1495-B {{template is declared here}} // Deduction fails in both directions. - template struct C {}; // expected-note {{template is declared here}} - template struct C<0, Ts...> {}; // expected-error {{not more specialized}} + template struct C {}; // #dr1495-C + template struct C<0, Ts...> {}; + // since-cxx11-error@-1 {{class template partial specialization is not more specialized than the primary template}} + // since-cxx11-note@#dr1495-C {{template is declared here}} #if __cplusplus >= 201402L // Deduction succeeds in both directions. - template int a; // expected-note {{template is declared here}} - template int a; // expected-error {{variable template partial specialization is not more specialized}} + template int a; // #dr1495-a + template int a; + // since-cxx14-error@-1 {{variable template partial specialization is not more specialized than the primary template}} + // since-cxx14-note@#dr1495-a {{template is declared here}} // Primary template is more specialized. - template int b; // expected-note {{template is declared here}} - template int b; // expected-error {{not more specialized}} + template int b; // #dr1495-b + template int b; + // since-cxx14-error@-1 {{variable template partial specialization is not more specialized than the primary template}} + // since-cxx14-note@#dr1495-b {{template is declared here}} // Deduction fails in both directions. - template int c; // expected-note {{template is declared here}} - template int c<0, Ts...>; // expected-error {{not more specialized}} + template int c; // #dr1495-c + template int c<0, Ts...>; + // since-cxx14-error@-1 {{variable template partial specialization is not more specialized than the primary template}} + // since-cxx14-note@#dr1495-c {{template is declared here}} +#endif #endif } @@ -544,5 +676,3 @@ struct A { static_assert(__is_trivial(A), ""); #endif } - -#endif diff --git a/clang/test/CXX/drs/dr15xx.cpp b/clang/test/CXX/drs/dr15xx.cpp index c588891a75d6..007b42c74aff 100644 --- a/clang/test/CXX/drs/dr15xx.cpp +++ b/clang/test/CXX/drs/dr15xx.cpp @@ -1,13 +1,18 @@ // RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,cxx14_17 -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,cxx11-14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,cxx11-14,cxx14-17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx17 -fexceptions -fcxx-exceptions -pedantic-errors namespace dr1512 { // dr1512: 4 void f(char *p) { - if (p > 0) {} // expected-error {{ordered comparison between pointer and zero}} + if (p > 0) {} + // expected-error@-1 {{ordered comparison between pointer and zero ('char *' and 'int')}} #if __cplusplus >= 201103L - if (p > nullptr) {} // expected-error {{invalid operands}} + if (p > nullptr) {} + // since-cxx11-error@-1 {{invalid operands to binary expression ('char *' and 'std::nullptr_t')}} #endif } bool g(int **x, const int **y) { @@ -28,10 +33,20 @@ namespace dr1512 { // dr1512: 4 template void composite_pointer_type_is_ord() { composite_pointer_type_is_base(); - typedef __typeof(val() < val()) cmp; // cxx17-warning 2 {{ordered comparison of function pointers}} - typedef __typeof(val() <= val()) cmp; // cxx17-warning 2 {{ordered comparison of function pointers}} - typedef __typeof(val() > val()) cmp; // cxx17-warning 2 {{ordered comparison of function pointers}} - typedef __typeof(val() >= val()) cmp; // cxx17-warning 2 {{ordered comparison of function pointers}} + typedef __typeof(val() < val()) cmp; // #dr1512-lt + // since-cxx17-warning@#dr1512-lt {{ordered comparison of function pointers ('int (*)() noexcept' and 'int (*)()')}} + // since-cxx17-note@#dr1512-noexcept-1st {{in instantiation of function template specialization 'dr1512::composite_pointer_type_is_ord' requested here}} + // since-cxx17-warning@#dr1512-lt {{ordered comparison of function pointers ('int (*)()' and 'int (*)() noexcept')}} + // since-cxx17-note@#dr1512-noexcept-2nd {{in instantiation of function template specialization 'dr1512::composite_pointer_type_is_ord' requested here}} + typedef __typeof(val() <= val()) cmp; + // since-cxx17-warning@-1 {{ordered comparison of function pointers ('int (*)() noexcept' and 'int (*)()')}} + // since-cxx17-warning@-2 {{ordered comparison of function pointers ('int (*)()' and 'int (*)() noexcept')}} + typedef __typeof(val() > val()) cmp; + // since-cxx17-warning@-1 {{ordered comparison of function pointers ('int (*)() noexcept' and 'int (*)()')}} + // since-cxx17-warning@-2 {{ordered comparison of function pointers ('int (*)()' and 'int (*)() noexcept')}} + typedef __typeof(val() >= val()) cmp; + // since-cxx17-warning@-1 {{ordered comparison of function pointers ('int (*)() noexcept' and 'int (*)()')}} + // since-cxx17-warning@-2 {{ordered comparison of function pointers ('int (*)()' and 'int (*)() noexcept')}} typedef bool cmp; } @@ -77,10 +92,11 @@ namespace dr1512 { // dr1512: 4 composite_pointer_type_is_unord(); no_composite_pointer_type(); no_composite_pointer_type(); + // since-cxx20-warning@-1 {{volatile-qualified return type 'volatile int' is deprecated}} -#if __cplusplus > 201402 - composite_pointer_type_is_ord(); // expected-note {{requested here}} - composite_pointer_type_is_ord(); // expected-note {{requested here}} +#if __cplusplus >= 201703L + composite_pointer_type_is_ord(); // #dr1512-noexcept-1st + composite_pointer_type_is_ord(); // #dr1512-noexcept-2nd composite_pointer_type_is_unord(); composite_pointer_type_is_unord(); // FIXME: This looks like a standard defect; these should probably all have type 'int (B::*)()'. @@ -113,24 +129,40 @@ namespace dr1512 { // dr1512: 4 } #if __cplusplus >= 201103L - template struct Wrap { operator T(); }; // expected-note 4{{converted to type 'std::nullptr_t'}} expected-note 4{{converted to type 'int *'}} + template struct Wrap { operator T(); }; // #dr1512-Wrap void test_overload() { using nullptr_t = decltype(nullptr); void(Wrap() == Wrap()); void(Wrap() != Wrap()); - void(Wrap() < Wrap()); // expected-error {{invalid operands}} - void(Wrap() > Wrap()); // expected-error {{invalid operands}} - void(Wrap() <= Wrap()); // expected-error {{invalid operands}} - void(Wrap() >= Wrap()); // expected-error {{invalid operands}} + void(Wrap() < Wrap()); + // since-cxx11-error@-1 {{invalid operands to binary expression ('Wrap' (aka 'Wrap') and 'Wrap' (aka 'Wrap'))}} + void(Wrap() > Wrap()); + // since-cxx11-error@-1 {{invalid operands to binary expression ('Wrap' (aka 'Wrap') and 'Wrap' (aka 'Wrap'))}} + void(Wrap() <= Wrap()); + // since-cxx11-error@-1 {{invalid operands to binary expression ('Wrap' (aka 'Wrap') and 'Wrap' (aka 'Wrap'))}} + void(Wrap() >= Wrap()); + // since-cxx11-error@-1 {{invalid operands to binary expression ('Wrap' (aka 'Wrap') and 'Wrap' (aka 'Wrap'))}} // Under dr1213, this is ill-formed: we select the builtin operator<(int*, int*) // but then only convert as far as 'nullptr_t', which we then can't convert to 'int*'. void(Wrap() == Wrap()); void(Wrap() != Wrap()); - void(Wrap() < Wrap()); // expected-error {{invalid operands to binary expression ('Wrap' (aka 'Wrap') and 'Wrap')}} - void(Wrap() > Wrap()); // expected-error {{invalid operands}} - void(Wrap() <= Wrap()); // expected-error {{invalid operands}} - void(Wrap() >= Wrap()); // expected-error {{invalid operands}} + void(Wrap() < Wrap()); + // since-cxx11-error@-1 {{invalid operands to binary expression ('Wrap' (aka 'Wrap') and 'Wrap')}} + // since-cxx11-note@#dr1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}} + // since-cxx11-note@#dr1512-Wrap {{second operand was implicitly converted to type 'int *'}} + void(Wrap() > Wrap()); + // since-cxx11-error@-1 {{invalid operands}} + // since-cxx11-note@#dr1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}} + // since-cxx11-note@#dr1512-Wrap{{second operand was implicitly converted to type 'int *'}} + void(Wrap() <= Wrap()); + // since-cxx11-error@-1 {{invalid operands}} + // since-cxx11-note@#dr1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}} + // since-cxx11-note@#dr1512-Wrap {{second operand was implicitly converted to type 'int *'}} + void(Wrap() >= Wrap()); + // since-cxx11-error@-1 {{invalid operands}} + // since-cxx11-note@#dr1512-Wrap {{first operand was implicitly converted to type 'std::nullptr_t'}} + // since-cxx11-note@#dr1512-Wrap {{second operand was implicitly converted to type 'int *'}} } #endif } @@ -138,8 +170,10 @@ namespace dr1512 { // dr1512: 4 namespace dr1514 { // dr1514: 11 #if __cplusplus >= 201103L struct S { - enum E : int {}; // expected-note {{previous}} - enum E : int {}; // expected-error {{redefinition}} + enum E : int {}; // #dr1514-E + enum E : int {}; + // since-cxx11-error@-1 {{redefinition of 'E'}} + // since-cxx11-note@#dr1514-E {{previous definition is here}} }; S::E se; // OK, complete type, not zero-width bitfield. @@ -149,91 +183,142 @@ namespace dr1514 { // dr1514: 11 namespace dr1518 { // dr1518: 4 #if __cplusplus >= 201103L -struct Z0 { // expected-note 0+ {{candidate}} - explicit Z0() = default; // expected-note 0+ {{here}} +struct Z0 { // #dr1518-Z0 + explicit Z0() = default; // #dr1518-Z0-ctor }; -struct Z { // expected-note 0+ {{candidate}} - explicit Z(); // expected-note 0+ {{here}} - explicit Z(int); // expected-note {{not a candidate}} - explicit Z(int, int); // expected-note 0+ {{here}} +struct Z { // #dr1518-Z + explicit Z(); // #dr1518-Z-ctor + explicit Z(int); // #dr1518-Z-int + explicit Z(int, int); // #dr1518-Z-int-int }; -template int Eat(T); // expected-note 0+ {{candidate}} +template int Eat(T); // #dr1518-Eat Z0 a; Z0 b{}; -Z0 c = {}; // expected-error {{explicit in copy-initialization}} -int i = Eat({}); // expected-error {{no matching function for call to 'Eat'}} - -Z c2 = {}; // expected-error {{explicit in copy-initialization}} -int i2 = Eat({}); // expected-error {{no matching function for call to 'Eat'}} -Z a1 = 1; // expected-error {{no viable conversion}} +Z0 c = {}; +// since-cxx11-error@-1 {{chosen constructor is explicit in copy-initialization}} +// since-cxx11-note@#dr1518-Z0-ctor {{explicit constructor declared here}} +int i = Eat({}); +// since-cxx11-error@-1 {{no matching function for call to 'Eat'}} +// since-cxx11-note@#dr1518-Eat {{candidate function template not viable: cannot convert initializer list argument to 'Z0'}} + +Z c2 = {}; +// since-cxx11-error@-1 {{chosen constructor is explicit in copy-initialization}} +// since-cxx11-note@#dr1518-Z-ctor {{explicit constructor declared here}} +int i2 = Eat({}); +// since-cxx11-error@-1 {{no matching function for call to 'Eat'}} +// since-cxx11-note@#dr1518-Eat {{candidate function template not viable: cannot convert initializer list argument to 'Z'}} +Z a1 = 1; +// since-cxx11-error@-1 {{no viable conversion from 'int' to 'Z'}} +// since-cxx11-note@#dr1518-Z {{candidate constructor (the implicit copy constructor) not viable: no known conversion from 'int' to 'const Z &' for 1st argument}} +// since-cxx11-note@#dr1518-Z {{candidate constructor (the implicit move constructor) not viable: no known conversion from 'int' to 'Z &&' for 1st argument}} +// since-cxx11-note@#dr1518-Z-int {{explicit constructor is not a candidate}} Z a3 = Z(1); Z a2(1); Z *p = new Z(1); Z a4 = (Z)1; Z a5 = static_cast(1); -Z a6 = {4, 3}; // expected-error {{explicit in copy-initialization}} +Z a6 = {4, 3}; +// since-cxx11-error@-1 {{chosen constructor is explicit in copy-initialization}} +// since-cxx11-note@#dr1518-Z-int-int {{explicit constructor declared here}} -struct UserProvidedBaseCtor { // expected-note 0+ {{candidate}} +struct UserProvidedBaseCtor { // #dr1518-U UserProvidedBaseCtor() {} }; -struct DoesntInheritCtor : UserProvidedBaseCtor { // expected-note 0+ {{candidate}} +struct DoesntInheritCtor : UserProvidedBaseCtor { // #dr1518-D-U int x; }; DoesntInheritCtor I{{}, 42}; -#if __cplusplus <= 201402L -// expected-error@-2 {{no matching constructor}} -#endif - -struct BaseCtor { BaseCtor() = default; }; // expected-note 0+ {{candidate}} -struct InheritsCtor : BaseCtor { // expected-note 1+ {{candidate}} - using BaseCtor::BaseCtor; // expected-note 2 {{inherited here}} +// cxx11-14-error@-1 {{no matching constructor for initialization of 'DoesntInheritCtor'}} +// cxx11-14-note@#dr1518-D-U {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}} +// cxx11-14-note@#dr1518-D-U {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}} +// cxx11-14-note@#dr1518-D-U {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 2 were provided}} + +struct BaseCtor { BaseCtor() = default; }; // #dr1518-BC +struct InheritsCtor : BaseCtor { // #dr1518-I + using BaseCtor::BaseCtor; // #dr1518-I-using int x; }; -InheritsCtor II = {{}, 42}; // expected-error {{no matching constructor}} +InheritsCtor II = {{}, 42}; +// since-cxx11-error@-1 {{no matching constructor for initialization of 'InheritsCtor'}} +// since-cxx11-note@#dr1518-BC {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}} +// since-cxx11-note@#dr1518-I-using {{constructor from base class 'BaseCtor' inherited here}} +// since-cxx11-note@#dr1518-BC {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}} +// since-cxx11-note@#dr1518-I-using {{constructor from base class 'BaseCtor' inherited here}} +// since-cxx11-note@#dr1518-I {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}} +// since-cxx11-note@#dr1518-I {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}} +// since-cxx11-note@#dr1518-I {{candidate constructor (the implicit default constructor) not viable: requires 0 arguments, but 2 were provided}} namespace std_example { struct A { - explicit A() = default; // expected-note 2{{declared here}} + explicit A() = default; // #dr1518-A }; struct B : A { - explicit B() = default; // expected-note 2{{declared here}} + explicit B() = default; // #dr1518-B }; struct C { - explicit C(); // expected-note 2{{declared here}} + explicit C(); // #dr1518-C }; struct D : A { C c; - explicit D() = default; // expected-note 2{{declared here}} + explicit D() = default; // #dr1518-D }; template void f() { T t; // ok T u{}; // ok - T v = {}; // expected-error 4{{explicit}} + T v = {}; // #dr1518-v + // since-cxx11-error@#dr1518-v {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-f-A {{in instantiation of function template specialization 'dr1518::std_example::f' requested here}} + // since-cxx11-note@#dr1518-A {{explicit constructor declared here}} + // since-cxx11-error@#dr1518-v {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-f-B {{in instantiation of function template specialization 'dr1518::std_example::f' requested here}} + // since-cxx11-note@#dr1518-B {{explicit constructor declared here}} + // since-cxx11-error@#dr1518-v {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-f-C {{in instantiation of function template specialization 'dr1518::std_example::f' requested here}} + // since-cxx11-note@#dr1518-C {{explicit constructor declared here}} + // since-cxx11-error@#dr1518-v {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-f-D {{in instantiation of function template specialization 'dr1518::std_example::f' requested here}} + // since-cxx11-note@#dr1518-D {{explicit constructor declared here}} } template void g() { - void x(T t); // expected-note 4{{parameter}} - x({}); // expected-error 4{{explicit}} + void x(T t); // #dr1518-x + x({}); // #dr1518-x-call + // since-cxx11-error@#dr1518-x-call {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-g-A {{in instantiation of function template specialization 'dr1518::std_example::g' requested here}} + // since-cxx11-note@#dr1518-A {{explicit constructor declared here}} + // since-cxx11-note@#dr1518-x {{passing argument to parameter 't' here}} + // since-cxx11-error@#dr1518-x-call {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-g-B {{in instantiation of function template specialization 'dr1518::std_example::g' requested here}} + // since-cxx11-note@#dr1518-B {{explicit constructor declared here}} + // since-cxx11-note@#dr1518-x {{passing argument to parameter 't' here}} + // since-cxx11-error@#dr1518-x-call {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-g-C {{in instantiation of function template specialization 'dr1518::std_example::g' requested here}} + // since-cxx11-note@#dr1518-C {{explicit constructor declared here}} + // since-cxx11-note@#dr1518-x {{passing argument to parameter 't' here}} + // since-cxx11-error@#dr1518-x-call {{chosen constructor is explicit in copy-initialization}} + // since-cxx11-note@#dr1518-g-D {{in instantiation of function template specialization 'dr1518::std_example::g' requested here}} + // since-cxx11-note@#dr1518-D {{explicit constructor declared here}} + // since-cxx11-note@#dr1518-x {{passing argument to parameter 't' here}} } void test() { - f(); // expected-note {{instantiation of}} - f(); // expected-note {{instantiation of}} - f(); // expected-note {{instantiation of}} - f(); // expected-note {{instantiation of}} - g(); // expected-note {{instantiation of}} - g(); // expected-note {{instantiation of}} - g(); // expected-note {{instantiation of}} - g(); // expected-note {{instantiation of}} + f(); // #dr1518-f-A + f(); // #dr1518-f-B + f(); // #dr1518-f-C + f(); // #dr1518-f-D + g(); // #dr1518-g-A + g(); // #dr1518-g-B + g(); // #dr1518-g-C + g(); // #dr1518-g-D } } -#endif // __cplusplus >= 201103L +#endif // __cplusplus >= 201103L } -namespace dr1550 { // dr1550: yes +namespace dr1550 { // dr1550: 3.4 int f(bool b, int n) { return (b ? (throw 0) : n) + (b ? n : (throw 0)); } @@ -242,13 +327,16 @@ namespace dr1550 { // dr1550: yes namespace dr1558 { // dr1558: 12 #if __cplusplus >= 201103L template using first_of = T; - template first_of f(int); // expected-note {{'int' cannot be used prior to '::'}} - template void f(...) = delete; // expected-note {{deleted}} + template first_of f(int); // #dr1558-f + template void f(...) = delete; // #dr1558-f-deleted struct X { typedef void type; }; void test() { f(0); - f(0); // expected-error {{deleted}} + f(0); + // since-cxx11-error@-1 {{call to deleted function 'f'}} + // since-cxx11-note@#dr1558-f-deleted {{candidate function [with T = int] has been explicitly deleted}} + // since-cxx11-note@#dr1558-f {{candidate template ignored: substitution failure [with T = int]: type 'int' cannot be used prior to '::' because it has no members}} } #endif } @@ -283,17 +371,25 @@ namespace dr1573 { // dr1573: 3.9 struct C { C(); constexpr C(int) {} }; struct D : C { using C::C; }; constexpr D d = D(0); // ok - struct E : C { using C::C; A a; }; // expected-note {{non-literal type}} - constexpr E e = E(0); // expected-error {{non-literal type}} + struct E : C { using C::C; A a; }; // #dr1573-E + constexpr E e = E(0); + // since-cxx11-error@-1 {{constexpr variable cannot have non-literal type 'const E'}} + // since-cxx11-note@#dr1573-E {{'E' is not literal because it has data member 'a' of non-literal type 'A'}} + // FIXME: This diagnostic is pretty bad; we should explain that the problem // is that F::c would be initialized by a non-constexpr constructor. - struct F : C { using C::C; C c; }; // expected-note {{here}} - constexpr F f = F(0); // expected-error {{constant expression}} expected-note {{constructor inherited from base class 'C'}} + struct F : C { using C::C; C c; }; // #dr1573-F + constexpr F f = F(0); + // since-cxx11-error@-1 {{constexpr variable 'f' must be initialized by a constant expression}} + // since-cxx11-note@-2 {{constructor inherited from base class 'C' cannot be used in a constant expression; derived class cannot be implicitly initialized}} + // since-cxx11-note@#dr1573-F {{declared here}} // inherited constructor is effectively deleted if the user-written constructor would be struct G { G(int); }; - struct H : G { using G::G; G g; }; // expected-note {{constructor inherited by 'H' is implicitly deleted because field 'g' has no default constructor}} - H h(0); // expected-error {{constructor inherited by 'H' from base class 'G' is implicitly deleted}} + struct H : G { using G::G; G g; }; // #dr1573-H + H h(0); + // since-cxx11-error@-1 {{constructor inherited by 'H' from base class 'G' is implicitly deleted}} + // since-cxx11-note@#dr1573-H {{constructor inherited by 'H' is implicitly deleted because field 'g' has no default constructor}} #endif } @@ -336,13 +432,15 @@ namespace std { typedef basic_string string; } // std +#endif namespace dr1579 { // dr1579: 3.9 +#if __cplusplus >= 201103L template struct GenericMoveOnly { GenericMoveOnly(); - template GenericMoveOnly(const GenericMoveOnly &) = delete; // expected-note 5 {{marked deleted here}} - GenericMoveOnly(const int &) = delete; // expected-note 2 {{marked deleted here}} + template GenericMoveOnly(const GenericMoveOnly &) = delete; // #dr1579-deleted-U + GenericMoveOnly(const int &) = delete; // #dr1579-deleted-int template GenericMoveOnly(GenericMoveOnly &&); GenericMoveOnly(int &&); }; @@ -369,17 +467,29 @@ GenericMoveOnly DR1579_Ineligible(int &AnInt, extern GenericMoveOnly ExternMove; if (0) - return AnInt; // expected-error{{invokes a deleted function}} + return AnInt; + // since-cxx11-error@-1 {{conversion function from 'int' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-int {{'GenericMoveOnly' has been explicitly marked deleted here}} else if (0) - return GlobalMO; // expected-error{{invokes a deleted function}} + return GlobalMO; + // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly' has been explicitly marked deleted here}} else if (0) - return StaticMove; // expected-error{{invokes a deleted function}} + return StaticMove; + // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly' has been explicitly marked deleted here}} else if (0) - return ExternMove; // expected-error{{invokes a deleted function}} + return ExternMove; + // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly' has been explicitly marked deleted here}} else if (0) - return AnInt; // expected-error{{invokes a deleted function}} + return AnInt; + // since-cxx11-error@-1 {{conversion function from 'int' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-int {{'GenericMoveOnly' has been explicitly marked deleted here}} else - return CharMO; // expected-error{{invokes a deleted function}} + return CharMO; + // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly' has been explicitly marked deleted here}} } auto DR1579_lambda_valid = [](GenericMoveOnly mo) -> @@ -389,24 +499,34 @@ auto DR1579_lambda_valid = [](GenericMoveOnly mo) -> auto DR1579_lambda_invalid = []() -> GenericMoveOnly { static GenericMoveOnly mo; - return mo; // expected-error{{invokes a deleted function}} + return mo; + // since-cxx11-error@-1 {{conversion function from 'GenericMoveOnly' to 'GenericMoveOnly' invokes a deleted function}} + // since-cxx11-note@#dr1579-deleted-U {{'GenericMoveOnly' has been explicitly marked deleted here}} }; +#endif } // end namespace dr1579 namespace dr1584 { +#if __cplusplus >= 201103L // Deducing function types from cv-qualified types - template void f(const T *); // expected-note {{candidate template ignored}} + template void f(const T *); // #dr1584-f template void g(T *, const T * = 0); - template void h(T *) { T::error; } // expected-error {{no members}} + template void h(T *) { T::error; } + // since-cxx11-error@-1 {{type 'void ()' cannot be used prior to '::' because it has no members}} + // since-cxx11-note@#dr1584-h {{in instantiation of function template specialization 'dr1584::h' requested here}} template void h(const T *); void i() { - f(&i); // expected-error {{no matching function}} + f(&i); + // since-cxx11-error@-1 {{no matching function for call to 'f'}} + // since-cxx11-note@#dr1584-f {{candidate template ignored: could not match 'const T *' against 'void (*)()'}} g(&i); - h(&i); // expected-note {{here}} + h(&i); // #dr1584-h } +#endif } namespace dr1589 { // dr1589: 3.7 c++11 +#if __cplusplus >= 201103L // Ambiguous ranking of list-initialization sequences void f0(long, int=0); // Would makes selection of #0 ambiguous @@ -425,25 +545,35 @@ namespace dr1589 { // dr1589: 3.7 c++11 void g2() { f2({"foo","bar"}); } // chooses #4 namespace with_error { - void f0(long); // #0 - void f0(std::initializer_list); // #00 expected-note {{candidate function}} - void f0(std::initializer_list, int = 0); // expected-note {{candidate function}} - void g0() { f0({1L}); } // expected-error{{call to 'f0' is ambiguous}} - - void f1(int); // #1 - void f1(std::initializer_list); // #2 expected-note {{candidate function}} - void f1(std::initializer_list, int = 0); // expected-note {{candidate function}} - void g1() { f1({42}); } // expected-error{{call to 'f1' is ambiguous}} - - void f2(std::pair); // #3 - void f2(std::initializer_list); // #4 expected-note {{candidate function}} - void f2(std::initializer_list, int = 0); // expected-note {{candidate function}} - void g2() { f2({"foo","bar"}); } // expected-error{{call to 'f2' is ambiguous}} + void f0(long); + void f0(std::initializer_list); // #dr1589-f0-ilist + void f0(std::initializer_list, int = 0); // #dr1589-f0-ilist-int + void g0() { f0({1L}); } + // since-cxx11-error@-1 {{call to 'f0' is ambiguous}} + // since-cxx11-note@#dr1589-f0-ilist {{candidate function}} + // since-cxx11-note@#dr1589-f0-ilist-int {{candidate function}} + + void f1(int); + void f1(std::initializer_list); // #dr1589-f1-ilist + void f1(std::initializer_list, int = 0); // #dr1589-f1-ilist-long + void g1() { f1({42}); } + // since-cxx11-error@-1 {{call to 'f1' is ambiguous}} + // since-cxx11-note@#dr1589-f1-ilist {{candidate function}} + // since-cxx11-note@#dr1589-f1-ilist-long {{candidate function}} + + void f2(std::pair); + void f2(std::initializer_list); // #dr1589-f2-ilist + void f2(std::initializer_list, int = 0); // #dr1589-f2-ilist-int + void g2() { f2({"foo","bar"}); } + // since-cxx11-error@-1 {{call to 'f2' is ambiguous}} + // since-cxx11-note@#dr1589-f2-ilist {{candidate function}} + // since-cxx11-note@#dr1589-f2-ilist-int {{candidate function}} } - +#endif } // dr1589 -namespace dr1591 { //dr1591. Deducing array bound and element type from initializer list +namespace dr1591 { //dr1591. Deducing array bound and element type from initializer list +#if __cplusplus >= 201103L template int h(T const(&)[N]); int X = h({1,2,3}); // T deduced to int, N deduced to 3 @@ -451,8 +581,10 @@ namespace dr1591 { //dr1591. Deducing array bound and element type from initial int Y = j({42}); // T deduced to int, array bound not considered struct Aggr { int i; int j; }; - template int k(Aggr const(&)[N]); //expected-note{{not viable}} - int Y0 = k({1,2,3}); //expected-error{{no matching function}} + template int k(Aggr const(&)[N]); // #dr1591-k + int Y0 = k({1,2,3}); + // since-cxx11-error@-1 {{no matching function for call to 'k'}} + // since-cxx11-note@#dr1591-k {{candidate function [with N = 3] not viable: no known conversion from 'int' to 'const Aggr' for 1st argument}} int Z = k({{1},{2},{3}}); // OK, N deduced to 3 template int m(int const(&)[M][N]); @@ -463,54 +595,64 @@ namespace dr1591 { //dr1591. Deducing array bound and element type from initial namespace check_multi_dim_arrays { - template int ***f(const T (&a)[N][M][O]); //expected-note{{deduced conflicting values}} - template int **f(const T (&a)[N][M]); //expected-note{{couldn't infer}} + template int ***f(const T (&a)[N][M][O]); // #dr1591-f-3 + template int **f(const T (&a)[N][M]); // #dr1591-f-2 - template int *f(const T (&a)[N]); //expected-note{{couldn't infer}} + template int *f(const T (&a)[N]); // #dr1591-f-1 int ***p3 = f({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12} } }); - int ***p33 = f({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12, 13} } }); //expected-error{{no matching}} + int ***p33 = f({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12, 13} } }); + // since-cxx11-error@-1 {{no matching function for call to 'f'}} + // since-cxx11-note@#dr1591-f-2 {{candidate template ignored: couldn't infer template argument 'T'}} + // since-cxx11-note@#dr1591-f-1 {{candidate template ignored: couldn't infer template argument 'T'}} + // since-cxx11-note@#dr1591-f-3 {{candidate template ignored: deduced conflicting values for parameter 'O' (2 vs. 3)}} int **p2 = f({ {1,2,3}, {3, 4, 5} }); int **p22 = f({ {1,2}, {3, 4} }); int *p1 = f({1, 2, 3}); } namespace check_multi_dim_arrays_rref { - template int ***f(T (&&a)[N][M][O]); //expected-note{{deduced conflicting values}} - template int **f(T (&&a)[N][M]); //expected-note{{couldn't infer}} + template int ***g(T (&&a)[N][M][O]); // #dr1591-g-3 + template int **g(T (&&a)[N][M]); // #dr1591-g-2 - template int *f(T (&&a)[N]); //expected-note{{couldn't infer}} - int ***p3 = f({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12} } }); - int ***p33 = f({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12, 13} } }); //expected-error{{no matching}} - int **p2 = f({ {1,2,3}, {3, 4, 5} }); - int **p22 = f({ {1,2}, {3, 4} }); - int *p1 = f({1, 2, 3}); + template int *g(T (&&a)[N]); // #dr1591-g-1 + int ***p3 = g({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12} } }); + int ***p33 = g({ { {1,2}, {3, 4} }, { {5,6}, {7, 8} }, { {9,10}, {11, 12, 13} } }); + // since-cxx11-error@-1 {{no matching function for call to 'g'}} + // since-cxx11-note@#dr1591-g-2 {{candidate template ignored: couldn't infer template argument 'T'}} + // since-cxx11-note@#dr1591-g-1 {{candidate template ignored: couldn't infer template argument 'T'}} + // since-cxx11-note@#dr1591-g-3 {{candidate template ignored: deduced conflicting values for parameter 'O' (2 vs. 3)}} + int **p2 = g({ {1,2,3}, {3, 4, 5} }); + int **p22 = g({ {1,2}, {3, 4} }); + int *p1 = g({1, 2, 3}); } namespace check_arrays_of_init_list { - template float *f(const std::initializer_list (&)[N]); - template double *f(const T(&)[N]); - double *p = f({1, 2, 3}); - float *fp = f({{1}, {1, 2}, {1, 2, 3}}); + template float *h(const std::initializer_list (&)[N]); + template double *h(const T(&)[N]); + double *p = h({1, 2, 3}); + float *fp = h({{1}, {1, 2}, {1, 2, 3}}); } namespace core_reflector_28543 { - template int *f(T (&&)[N]); // #1 - template char *f(std::initializer_list &&); //#2 - template int **f(T (&&)[N][M]); //#3 expected-note{{candidate}} - template char **f(std::initializer_list (&&)[N]); //#4 expected-note{{candidate}} + template int *i(T (&&)[N]); // #1 + template char *i(std::initializer_list &&); // #2 + template int **i(T (&&)[N][M]); // #3 #dr1591-i-2 + template char **i(std::initializer_list (&&)[N]); // #4 #dr1591-i-1 - template short *f(T (&&)[2]); //#5 + template short *i(T (&&)[2]); // #5 template using Arr = T[]; - char *pc = f({1, 2, 3}); // OK prefer #2 via 13.3.3.2 [over.ics.rank] - char *pc2 = f({1, 2}); // #2 also - int *pi = f(Arr{1, 2, 3}); // OK prefer #1 + char *pc = i({1, 2, 3}); // OK prefer #2 via 13.3.3.2 [over.ics.rank] + char *pc2 = i({1, 2}); // #2 also + int *pi = i(Arr{1, 2, 3}); // OK prefer #1 - void *pv1 = f({ {1, 2, 3}, {4, 5, 6} }); // expected-error{{ambiguous}} btw 3 & 4 - char **pcc = f({ {1}, {2, 3} }); // OK #4 + void *pv1 = i({ {1, 2, 3}, {4, 5, 6} }); // ambiguous btw 3 & 4 + // since-cxx11-error@-1 {{call to 'i' is ambiguous}} + // since-cxx11-note@#dr1591-i-2 {{candidate function [with T = int, N = 2, M = 3]}} + // since-cxx11-note@#dr1591-i-1 {{candidate function [with T = int, N = 2]}} + char **pcc = i({ {1}, {2, 3} }); // OK #4 - short *ps = f(Arr{1, 2}); // OK #5 + short *ps = i(Arr{1, 2}); // OK #5 } -} // dr1591 - #endif +} // dr1591 diff --git a/clang/test/CXX/drs/dr16xx.cpp b/clang/test/CXX/drs/dr16xx.cpp index 463353dab3b1..3f074c4d5735 100644 --- a/clang/test/CXX/drs/dr16xx.cpp +++ b/clang/test/CXX/drs/dr16xx.cpp @@ -1,12 +1,14 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2a -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors - -#if __cplusplus < 201103L -// expected-error@+1 {{variadic macro}} +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,cxx98-14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx14,since-cxx20,since-cxx17,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors + +#if __cplusplus == 199711L #define static_assert(...) __extension__ _Static_assert(__VA_ARGS__) +// cxx98-error@-1 {{variadic macros are a C99 feature}} #endif #if __cplusplus >= 201103L @@ -25,9 +27,7 @@ namespace std { namespace dr1601 { // dr1601: 10 enum E : char { e }; -#if __cplusplus < 201103L - // expected-error@-2 {{enumeration types with a fixed underlying type are a C++11 extension}} -#endif +// cxx98-error@-1 {{enumeration types with a fixed underlying type are a C++11 extension}} void f(char); void f(int); void g() { @@ -53,26 +53,30 @@ namespace dr1631 { // dr1631: 3.7 void f(int, A); void test() { - f({0}, {{1}}); // expected-warning {{braces around scalar init}} + f({0}, {{1}}); + // since-cxx11-warning@-1 {{braces around scalar initializer}} } namespace with_error { void f(B, int); // TODO: expected- note {{candidate function}} - void f(int, A); // expected-note {{candidate function}} - void f(int, A, int = 0); // expected-note {{candidate function}} + void f(int, A); // #dr1631-f + void f(int, A, int = 0); // #dr1631-f-int void test() { - f({0}, {{1}}); // expected-error{{call to 'f' is ambiguous}} + f({0}, {{1}}); + // since-cxx11-error@-1 {{call to 'f' is ambiguous}} + // since-cxx11-note@#dr1631-f {{candidate function}} + // since-cxx11-note@#dr1631-f-int {{candidate function}} } } #endif } -namespace dr1638 { // dr1638: yes +namespace dr1638 { // dr1638: 3.1 #if __cplusplus >= 201103L template struct A { - enum class E; // expected-note {{previous}} - enum class F : T; // expected-note 2{{previous}} + enum class E; // #dr1638-E + enum class F : T; // #dr1638-F }; template<> enum class A::E; @@ -83,16 +87,27 @@ namespace dr1638 { // dr1638: yes template<> enum class A::E : int; template<> enum class A::E : int {}; - template<> enum class A::F; // expected-error {{different underlying type}} - template<> enum class A::E : char; // expected-error {{different underlying type}} - template<> enum class A::F : int; // expected-error {{different underlying type}} - - enum class A::E; // expected-error {{template specialization requires 'template<>'}} expected-error {{nested name specifier}} - template enum class A::E; // expected-error {{enumerations cannot be explicitly instantiated}} - enum class A::E *e; // expected-error {{must use 'enum' not 'enum class'}} + template<> enum class A::F; + // since-cxx11-error@-1 {{enumeration redeclared with different underlying type 'int' (was 'short')}} + // since-cxx11-note@#dr1638-F {{previous declaration is here}} + template<> enum class A::E : char; + // since-cxx11-error@-1 {{enumeration redeclared with different underlying type 'char' (was 'int')}} + // since-cxx11-note@#dr1638-E {{previous declaration is here}} + template<> enum class A::F : int; + // since-cxx11-error@-1 {{enumeration redeclared with different underlying type 'int' (was 'char')}} + // since-cxx11-note@#dr1638-F {{previous declaration is here}} + + enum class A::E; + // since-cxx11-error@-1 {{template specialization requires 'template<>'}} + // since-cxx11-error@-2 {{forward declaration of enum class cannot have a nested name specifier}} + template enum class A::E; + // since-cxx11-error@-1 {{enumerations cannot be explicitly instantiated}} + enum class A::E *e; + // since-cxx11-error@-1 {{reference to enumeration must use 'enum' not 'enum class'}} struct B { - friend enum class A::E; // expected-error {{must use 'enum' not 'enum class'}} + friend enum class A::E; + // since-cxx11-error@-1 {{reference to enumeration must use 'enum' not 'enum class'}} }; #endif } @@ -100,37 +115,50 @@ namespace dr1638 { // dr1638: yes namespace dr1645 { // dr1645: 3.9 #if __cplusplus >= 201103L struct A { - constexpr A(int, float = 0); // expected-note {{candidate}} - explicit A(int, int = 0); // expected-note 2{{candidate}} - A(int, int, int = 0) = delete; // expected-note {{candidate}} + constexpr A(int, float = 0); // #dr1645-int-float + explicit A(int, int = 0); // #dr1645-int-int + A(int, int, int = 0) = delete; // #dr1645-int-int-int }; struct B : A { - using A::A; // expected-note 4{{inherited here}} + using A::A; // #dr1645-using }; - constexpr B a(0); // expected-error {{ambiguous}} - constexpr B b(0, 0); // expected-error {{ambiguous}} + constexpr B a(0); + // since-cxx11-error@-1 {{call to constructor of 'const B' is ambiguous}} + // since-cxx11-note@#dr1645-int-float {{candidate inherited constructor}} + // since-cxx11-note@#dr1645-using {{constructor from base class 'A' inherited here}} + // since-cxx11-note@#dr1645-int-int {{candidate inherited constructor}} + // since-cxx11-note@#dr1645-using {{constructor from base class 'A' inherited here}} + constexpr B b(0, 0); + // since-cxx11-error@-1 {{call to constructor of 'const B' is ambiguous}} + // since-cxx11-note@#dr1645-int-int {{candidate inherited constructor}} + // since-cxx11-note@#dr1645-using {{constructor from base class 'A' inherited here}} + // since-cxx11-note@#dr1645-int-int-int {{candidate inherited constructor has been explicitly deleted}} + // since-cxx11-note@#dr1645-using {{constructor from base class 'A' inherited here}} #endif } namespace dr1652 { // dr1652: 3.6 int a, b; - int arr[&a + 1 == &b ? 1 : 2]; // expected-error 2{{variable length array}} - // expected-note@-1 {{points past the end}} + int arr[&a + 1 == &b ? 1 : 2]; + // expected-error@-1 {{variable length arrays in C++ are a Clang extension}} + // expected-note@-2 {{comparison against pointer '&a + 1' that points past the end of a complete object has unspecified value}} + // expected-error@-3 {{variable length array declaration not allowed at file scope}} } namespace dr1653 { // dr1653: 4 c++17 void f(bool b) { ++b; + // cxx98-14-warning@-1 {{incrementing expression of type bool is deprecated and incompatible with C++17}} + // since-cxx17-error@-2 {{SO C++17 does not allow incrementing expression of type bool}} b++; -#if __cplusplus <= 201402L - // expected-warning@-3 {{deprecated}} expected-warning@-2 {{deprecated}} -#else - // expected-error@-5 {{incrementing expression of type bool}} expected-error@-4 {{incrementing expression of type bool}} -#endif - --b; // expected-error {{cannot decrement expression of type bool}} - b--; // expected-error {{cannot decrement expression of type bool}} + // cxx98-14-warning@-1 {{incrementing expression of type bool is deprecated and incompatible with C++17}} + // since-cxx17-error@-2 {{SO C++17 does not allow incrementing expression of type bool}} + --b; + // expected-error@-1 {{cannot decrement expression of type bool}} + b--; + // expected-error@-1 {{cannot decrement expression of type bool}} b += 1; // ok b -= 1; // ok } @@ -138,71 +166,88 @@ namespace dr1653 { // dr1653: 4 c++17 namespace dr1658 { // dr1658: 5 namespace DefCtor { - class A { A(); }; // expected-note 0-2{{here}} - class B { ~B(); }; // expected-note 0-2{{here}} + class A { A(); }; // #dr1658-A1 + class B { ~B(); }; // #dr1658-B1 // The stars align! An abstract class does not construct its virtual bases. struct C : virtual A { C(); virtual void foo() = 0; }; - C::C() = default; // ok, not deleted, expected-error 0-1{{extension}} + C::C() = default; // ok, not deleted + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} struct D : virtual B { D(); virtual void foo() = 0; }; - D::D() = default; // ok, not deleted, expected-error 0-1{{extension}} + D::D() = default; // ok, not deleted + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} // In all other cases, we are not so lucky. - struct E : A { E(); virtual void foo() = 0; }; -#if __cplusplus < 201103L - E::E() = default; // expected-error {{private default constructor}} expected-error {{extension}} expected-note {{here}} -#else - E::E() = default; // expected-error {{would delete}} expected-note@-4{{inaccessible default constructor}} -#endif - struct F : virtual A { F(); }; -#if __cplusplus < 201103L - F::F() = default; // expected-error {{private default constructor}} expected-error {{extension}} expected-note {{here}} -#else - F::F() = default; // expected-error {{would delete}} expected-note@-4{{inaccessible default constructor}} -#endif - - struct G : B { G(); virtual void foo() = 0; }; -#if __cplusplus < 201103L - G::G() = default; // expected-error@-2 {{private destructor}} expected-error {{extension}} expected-note {{here}} -#else - G::G() = default; // expected-error {{would delete}} expected-note@-4{{inaccessible destructor}} -#endif - struct H : virtual B { H(); }; -#if __cplusplus < 201103L - H::H() = default; // expected-error@-2 {{private destructor}} expected-error {{extension}} expected-note {{here}} -#else - H::H() = default; // expected-error {{would delete}} expected-note@-4{{inaccessible destructor}} -#endif + struct E : A { E(); virtual void foo() = 0; }; // #dr1658-E1 + E::E() = default; // #dr1658-E1-ctor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-2 {{base class 'A' has private default constructor}} + // cxx98-note@-3 {{in defaulted default constructor for 'dr1658::DefCtor::E' first required here}} + // cxx98-note@#dr1658-A1 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-E1-ctor {{defaulting this default constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-E1 {{default constructor of 'E' is implicitly deleted because base class 'A' has an inaccessible default constructor}} + struct F : virtual A { F(); }; // #dr1658-F1 + F::F() = default; // #dr1658-F1-ctor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-2 {{inherited virtual base class 'A' has private default constructor}} + // cxx98-note@-3 {{in defaulted default constructor for 'dr1658::DefCtor::F' first required here}} + // cxx98-note@#dr1658-A1 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-F1-ctor {{defaulting this default constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-F1 {{default constructor of 'F' is implicitly deleted because base class 'A' has an inaccessible default constructor}} + + struct G : B { G(); virtual void foo() = 0; }; // #dr1658-G1 + G::G() = default; // #dr1658-G1-ctor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@#dr1658-G1 {{base class 'B' has private destructor}} + // cxx98-note@#dr1658-G1-ctor {{in defaulted default constructor for 'dr1658::DefCtor::G' first required here}} + // cxx98-note@#dr1658-B1 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-G1-ctor {{defaulting this default constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-G1 {{default constructor of 'G' is implicitly deleted because base class 'B' has an inaccessible destructor}} + struct H : virtual B { H(); }; // #dr1658-H1 + H::H() = default; // #dr1658-H1-ctor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@#dr1658-H1 {{base class 'B' has private destructor}} + // cxx98-note@#dr1658-H1-ctor {{in defaulted default constructor for 'dr1658::DefCtor::H' first required here}} + // cxx98-note@#dr1658-B1 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-H1-ctor {{defaulting this default constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-H1 {{default constructor of 'H' is implicitly deleted because base class 'B' has an inaccessible destructor}} } namespace Dtor { - class B { ~B(); }; // expected-note 0-2{{here}} + class B { ~B(); }; // #dr1658-B2 struct D : virtual B { ~D(); virtual void foo() = 0; }; - D::~D() = default; // ok, not deleted, expected-error 0-1{{extension}} - - struct G : B { ~G(); virtual void foo() = 0; }; -#if __cplusplus < 201103L - G::~G() = default; // expected-error@-2 {{private destructor}} expected-error {{extension}} expected-note {{here}} -#else - G::~G() = default; // expected-error {{would delete}} expected-note@-4{{inaccessible destructor}} -#endif - struct H : virtual B { ~H(); }; -#if __cplusplus < 201103L - H::~H() = default; // expected-error@-2 {{private destructor}} expected-error {{extension}} expected-note {{here}} -#else - H::~H() = default; // expected-error {{would delete}} expected-note@-4{{inaccessible destructor}} -#endif + D::~D() = default; // ok, not deleted + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + + struct G : B { ~G(); virtual void foo() = 0; }; // #dr1658-G2 + G::~G() = default; // #dr1658-G2-dtor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@#dr1658-G2 {{base class 'B' has private destructor}} + // cxx98-note@#dr1658-G2-dtor {{in defaulted destructor for 'dr1658::Dtor::G' first required here}} + // cxx98-note@#dr1658-B2 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-G2-dtor {{defaulting this destructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-G2 {{destructor of 'G' is implicitly deleted because base class 'B' has an inaccessible destructor}} + struct H : virtual B { ~H(); }; // #dr1658-H2 + H::~H() = default; // #dr1658-H2-dtor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@#dr1658-H2 {{base class 'B' has private destructor}} + // cxx98-note@#dr1658-H2-dtor {{in defaulted destructor for 'dr1658::Dtor::H' first required here}} + // cxx98-note@#dr1658-B2 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-H2-dtor {{defaulting this destructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-H2 {{destructor of 'H' is implicitly deleted because base class 'B' has an inaccessible destructor}} } namespace MemInit { - struct A { A(int); }; // expected-note {{here}} + struct A { A(int); }; // #dr1658-A3 struct B : virtual A { B() {} virtual void f() = 0; }; struct C : virtual A { - C() {} // expected-error {{must explicitly initialize}} + C() {} + // expected-error@-1 {{constructor for 'dr1658::MemInit::C' must explicitly initialize the base class 'A' which does not have a default constructor}} + // expected-note@#dr1658-A3 {{'dr1658::MemInit::A' declared here}} }; } @@ -220,28 +265,51 @@ namespace dr1658 { // dr1658: 5 } namespace CopyCtor { - class A { A(const A&); A(A&&); }; // expected-note 0-4{{here}} expected-error 0-1{{extension}} - - struct C : virtual A { C(const C&); C(C&&); virtual void foo() = 0; }; // expected-error 0-1{{extension}} - C::C(const C&) = default; // expected-error 0-1{{extension}} - C::C(C&&) = default; // expected-error 0-2{{extension}} - - struct E : A { E(const E&); E(E&&); virtual void foo() = 0; }; // expected-error 0-1{{extension}} -#if __cplusplus < 201103L - E::E(const E&) = default; // expected-error {{private copy constructor}} expected-error {{extension}} expected-note {{here}} - E::E(E&&) = default; // expected-error {{private move constructor}} expected-error 2{{extension}} expected-note {{here}} -#else - E::E(const E&) = default; // expected-error {{would delete}} expected-note@-5{{inaccessible copy constructor}} - E::E(E&&) = default; // expected-error {{would delete}} expected-note@-6{{inaccessible move constructor}} -#endif - struct F : virtual A { F(const F&); F(F&&); }; // expected-error 0-1{{extension}} -#if __cplusplus < 201103L - F::F(const F&) = default; // expected-error {{private copy constructor}} expected-error {{extension}} expected-note {{here}} - F::F(F&&) = default; // expected-error {{private move constructor}} expected-error 2{{extension}} expected-note {{here}} -#else - F::F(const F&) = default; // expected-error {{would delete}} expected-note@-5{{inaccessible copy constructor}} - F::F(F&&) = default; // expected-error {{would delete}} expected-note@-6{{inaccessible move constructor}} -#endif + class A { A(const A&); A(A&&); }; // #dr1658-A5 + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + + struct C : virtual A { C(const C&); C(C&&); virtual void foo() = 0; }; + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + C::C(const C&) = default; + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + C::C(C&&) = default; + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + // cxx98-error@-2 {{defaulted function definitions are a C++11 extension}} + + struct E : A { E(const E&); E(E&&); virtual void foo() = 0; }; // #dr1658-E5 + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + E::E(const E&) = default; // #dr1658-E5-copy-ctor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-2 {{base class 'A' has private copy constructor}} + // cxx98-note@-3 {{in defaulted copy constructor for 'dr1658::CopyCtor::E' first required here}} + // cxx98-note@#dr1658-A5 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-E5-copy-ctor {{defaulting this copy constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-E5 {{copy constructor of 'E' is implicitly deleted because base class 'A' has an inaccessible copy constructor}} + E::E(E&&) = default; // #dr1658-E5-move-ctor + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + // cxx98-error@-2 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-3 {{base class 'A' has private move constructor}} + // cxx98-note@-4 {{in defaulted move constructor for 'dr1658::CopyCtor::E' first required here}} + // cxx98-note@#dr1658-A5 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-E5-move-ctor {{defaulting this move constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-E5 {{move constructor of 'E' is implicitly deleted because base class 'A' has an inaccessible move constructor}} + struct F : virtual A { F(const F&); F(F&&); }; // #dr1658-F5 + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + F::F(const F&) = default; // #dr1658-F5-copy-ctor + // cxx98-error@-1 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-2 {{inherited virtual base class 'A' has private copy constructor}} + // cxx98-note@-3 {{in defaulted copy constructor for 'dr1658::CopyCtor::F' first required here}} + // cxx98-note@#dr1658-A5 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-F5-copy-ctor {{defaulting this copy constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-F5 {{copy constructor of 'F' is implicitly deleted because base class 'A' has an inaccessible copy constructor}} + F::F(F&&) = default; // #dr1658-F5-move-ctor + // cxx98-error@-1 {{rvalue references are a C++11 extension}} + // cxx98-error@-2 {{defaulted function definitions are a C++11 extension}} + // cxx98-error@-3 {{inherited virtual base class 'A' has private move constructor}} + // cxx98-note@-4 {{in defaulted move constructor for 'dr1658::CopyCtor::F' first required here}} + // cxx98-note@#dr1658-A5 {{implicitly declared private here}} + // since-cxx11-error@#dr1658-F5-move-ctor {{defaulting this move constructor would delete it after its first declaration}} + // since-cxx11-note@#dr1658-F5 {{move constructor of 'F' is implicitly deleted because base class 'A' has an inaccessible move constructor}} } // assignment case is superseded by dr2180 @@ -274,31 +342,38 @@ namespace dr1672 { // dr1672: 7 namespace dr1684 { // dr1684: 3.6 #if __cplusplus >= 201103L - struct NonLiteral { // expected-note {{because}} + struct NonLiteral { // #dr1684-struct NonLiteral(); - constexpr int f() { return 0; } // expected-warning 0-1{{will not be implicitly 'const'}} + constexpr int f() { return 0; } + // cxx11-warning@-1 {{'constexpr' non-static member function will not be implicitly 'const' in C++14; add 'const' to avoid a change in behavior}} }; constexpr int f(NonLiteral &) { return 0; } - constexpr int f(NonLiteral) { return 0; } // expected-error {{not a literal type}} + constexpr int f(NonLiteral) { return 0; } + // since-cxx11-error@-1 {{constexpr function's 1st parameter type 'NonLiteral' is not a literal type}} + // since-cxx11-note@#dr1684-struct {{'NonLiteral' is not literal because it is not an aggregate and has no constexpr constructors other than copy or move constructors}} #endif } namespace dr1687 { // dr1687: 7 template struct To { - operator T(); // expected-note 2{{first operand was implicitly converted to type 'int *'}} - // expected-note@-1 {{second operand was implicitly converted to type 'double'}} -#if __cplusplus > 201703L - // expected-note@-3 2{{operand was implicitly converted to type 'dr1687::E}} -#endif + operator T(); // #dr1687-op-T }; - int *a = To() + 100.0; // expected-error {{invalid operands to binary expression ('To' and 'double')}} - int *b = To() + To(); // expected-error {{invalid operands to binary expression ('To' and 'To')}} + int *a = To() + 100.0; + // expected-error@-1 {{invalid operands to binary expression ('To' and 'double')}} + // expected-note@#dr1687-op-T {{first operand was implicitly converted to type 'int *'}} + // since-cxx20-note@#dr1687-op-T {{second operand was implicitly converted to type 'dr1687::E2'}} + int *b = To() + To(); + // expected-error@-1 {{invalid operands to binary expression ('To' and 'To')}} + // expected-note@#dr1687-op-T {{first operand was implicitly converted to type 'int *'}} + // expected-note@#dr1687-op-T {{second operand was implicitly converted to type 'double'}} -#if __cplusplus > 201703L +#if __cplusplus >= 202002L enum E1 {}; enum E2 {}; - auto c = To() <=> To(); // expected-error {{invalid operands to binary expression ('To' and 'To')}} + auto c = To() <=> To(); + // since-cxx20-error@-1 {{invalid operands to binary expression ('To' and 'To')}} + // since-cxx20-note@#dr1687-op-T {{operand was implicitly converted to type 'dr1687::E}} #endif } @@ -325,12 +400,14 @@ namespace dr1691 { // dr1691: 9 void f(E); } enum M::E : int {}; - void g(M::E); // expected-note {{declared here}} + void g(M::E); // #dr1691-g } void test() { N::M::E e; f(e); // ok - g(e); // expected-error {{use of undeclared}} + g(e); + // since-cxx11-error@-1 {{use of undeclared identifier 'g'; did you mean 'N::g'?}} + // since-cxx11-note@#dr1691-g {{'N::g' declared here}} } #endif } @@ -356,7 +433,9 @@ namespace dr1696 { // dr1696: 7 extern struct A a; struct A { const A &x = { A{a, a} }; - const A &y = { A{} }; // expected-error {{default member initializer for 'y' needed within definition of enclosing class 'A' outside of member functions}} expected-note {{here}} + const A &y = { A{} }; + // since-cxx14-error@-1 {{default member initializer for 'y' needed within definition of enclosing class 'A' outside of member functions}} + // since-cxx14-note@-2 {{default member initializer declared here}} }; A a{a, a}; #endif @@ -365,16 +444,20 @@ namespace dr1696 { // dr1696: 7 struct A { A(); ~A(); }; #if __cplusplus >= 201103L struct B { - A &&a; // expected-note {{declared here}} - B() : a{} {} // expected-error {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + A &&a; // #dr1696-a + B() : a{} {} + // since-cxx11-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-a {{reference member declared here}} } b; #endif struct C { C(); - const A &a; // expected-note {{declared here}} + const A &a; // #dr1696-C-a }; - C::C() : a(A()) {} // expected-error {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + C::C() : a(A()) {} + // expected-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // expected-note@#dr1696-C-a {{reference member declared here}} #if __cplusplus >= 201103L // This is OK in C++14 onwards, per DR1815, though we don't support that yet: @@ -383,51 +466,62 @@ namespace dr1696 { // dr1696: 7 // D1 d1 = {A()}; // ... which lifetime-extends the A temporary. struct D1 { -#if __cplusplus < 201402L - // expected-error@-2 {{binds to a temporary}} -#endif - const A &a = A(); // expected-note {{default member init}} + // cxx11-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // cxx11-note@#dr1696-d1 {{in implicit default constructor for 'dr1696::D1' first required here}} + // cxx11-note@#dr1696-D1-a {{initializing field 'a' with default member initializer}} + const A &a = A(); // #dr1696-D1-a }; - D1 d1 = {}; -#if __cplusplus < 201402L - // expected-note@-2 {{first required here}} -#else - // expected-warning-re@-4 {{sorry, lifetime extension {{.*}} not supported}} -#endif + D1 d1 = {}; // #dr1696-d1 + // since-cxx14-warning@-1 {{sorry, lifetime extension of temporary created by aggregate initialization using default member initializer is not supported; lifetime of temporary will end at the end of the full-expression}} + // since-cxx14-note@#dr1696-D1-a {{initializing field 'a' with default member initializer}} struct D2 { - const A &a = A(); // expected-note {{default member init}} - D2() {} // expected-error {{binds to a temporary}} + const A &a = A(); // #dr1696-D2-a + D2() {} + // since-cxx11-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-D2-a {{initializing field 'a' with default member initializer}} }; - struct D3 { // expected-error {{binds to a temporary}} - const A &a = A(); // expected-note {{default member init}} + struct D3 { + // since-cxx11-error@-1 {{reference member 'a' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-d3 {{in implicit default constructor for 'dr1696::D3' first required here}} + // since-cxx11-note@#dr1696-D3-a {{initializing field 'a' with default member initializer}} + const A &a = A(); // #dr1696-D3-a }; - D3 d3; // expected-note {{first required here}} + D3 d3; // #dr1696-d3 struct haslist1 { - std::initializer_list il; // expected-note {{'std::initializer_list' member}} - haslist1(int i) : il{i, 2, 3} {} // expected-error {{backing array for 'std::initializer_list' member 'il' is a temporary object}} + std::initializer_list il; // #dr1696-il-1 + haslist1(int i) : il{i, 2, 3} {} + // since-cxx11-error@-1 {{backing array for 'std::initializer_list' member 'il' is a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-il-1 {{'std::initializer_list' member declared here}} }; struct haslist2 { - std::initializer_list il; // expected-note {{'std::initializer_list' member}} + std::initializer_list il; // #dr1696-il-2 haslist2(); }; - haslist2::haslist2() : il{1, 2} {} // expected-error {{backing array for 'std::initializer_list' member 'il' is a temporary object}} + haslist2::haslist2() : il{1, 2} {} + // since-cxx11-error@-1 {{backing array for 'std::initializer_list' member 'il' is a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-il-2 {{'std::initializer_list' member declared here}} struct haslist3 { std::initializer_list il = {1, 2, 3}; }; - struct haslist4 { // expected-error {{backing array for 'std::initializer_list' member 'il' is a temporary object}} - std::initializer_list il = {1, 2, 3}; // expected-note {{default member initializer}} + struct haslist4 { + // since-cxx11-error@-1 {{backing array for 'std::initializer_list' member 'il' is a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-hl4 {{in implicit default constructor for 'dr1696::haslist4' first required here}} + // since-cxx11-note@#dr1696-il-4 {{initializing field 'il' with default member initializer}} + std::initializer_list il = {1, 2, 3}; // #dr1696-il-4 }; - haslist4 hl4; // expected-note {{in implicit default constructor}} + haslist4 hl4; // #dr1696-hl4 struct haslist5 { - std::initializer_list il = {1, 2, 3}; // expected-note {{default member initializer}} - haslist5() {} // expected-error {{backing array for 'std::initializer_list' member 'il' is a temporary object}} + std::initializer_list il = {1, 2, 3}; // #dr1696-il-5 + haslist5() {} + // since-cxx11-error@-1 {{backing array for 'std::initializer_list' member 'il' is a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx11-note@#dr1696-il-5 {{nitializing field 'il' with default member initializer}} }; #endif } diff --git a/clang/test/CXX/drs/dr17xx.cpp b/clang/test/CXX/drs/dr17xx.cpp index 219119d1a4cd..0c44fb231ce5 100644 --- a/clang/test/CXX/drs/dr17xx.cpp +++ b/clang/test/CXX/drs/dr17xx.cpp @@ -1,10 +1,10 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2c %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors namespace dr1710 { // dr1710: no // FIXME: all of the following is well-formed @@ -32,13 +32,17 @@ namespace dr1715 { // dr1715: 3.9 struct D : B { using B::B; }; - struct E : B { // expected-note 2{{candidate}} - template E(T t, typename T::Q q) : B(t, q) {} // expected-note {{'Q' is a private member}} + struct E : B { // #dr1715-E + template E(T t, typename T::Q q) : B(t, q) {} // #dr1715-E-ctor }; B b(S(), 1); D d(S(), 2); - E e(S(), 3); // expected-error {{no match}} + E e(S(), 3); + // since-cxx11-error@-1 {{no matching constructor for initialization of 'E'}} + // since-cxx11-note@#dr1715-E-ctor {{candidate template ignored: substitution failure [with T = S]: 'Q' is a private member of 'dr1715::S'}} + // since-cxx11-note@#dr1715-E {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 2 were provided}} + // since-cxx11-note@#dr1715-E {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 2 were provided}} #endif } @@ -73,12 +77,15 @@ struct S { struct L : S { using S::S; }; - typename T::type value; // expected-error {{no member}} - L l(value); // expected-note {{instantiation of}} + typename T::type value; + // since-cxx11-error@-1 {{type 'int' cannot be used prior to '::' because it has no members}} + // since-cxx11-note@#dr1736-l {{in instantiation of function template specialization 'dr1736::S::S' requested here}} + // since-cxx11-note@#dr1736-s {{in instantiation of function template specialization 'dr1736::S::S' requested here}} + L l(value); // #dr1736-l } }; struct Q { typedef int type; } q; -S s(q); // expected-note {{instantiation of}} +S s(q); // #dr1736-s #endif } @@ -91,18 +98,23 @@ namespace dr1753 { // dr1753: 11 n.~T(); n.T::~T(); - n.dr1753::~T(); // expected-error {{'dr1753' does not refer to a type name in pseudo-destructor}} + n.dr1753::~T(); + // expected-error@-1 {{'dr1753' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T' (aka 'int')}} n.dr1753::T::~T(); - n.A::~T(); // expected-error {{the type of object expression ('T' (aka 'int')) does not match the type being destroyed ('A') in pseudo-destructor expression}} + n.A::~T(); + // expected-error@-1 {{the type of object expression ('T' (aka 'int')) does not match the type being destroyed ('A') in pseudo-destructor expression}} n.A::T::~T(); - n.B::~T(); // expected-error {{'B' does not refer to a type name in pseudo-destructor expression}} + n.B::~T(); + // expected-error@-1 {{'B' does not refer to a type name in pseudo-destructor expression; expected the name of type 'T' (aka 'int')}} n.B::T::~T(); #if __cplusplus >= 201103L - n.decltype(n)::~T(); // expected-error {{not a class, namespace, or enumeration}} - n.T::~decltype(n)(); // expected-error {{expected a class name after '~'}} + n.decltype(n)::~T(); + // since-cxx11-error@-1 {{'decltype(n)' (aka 'int') is not a class, namespace, or enumeration}} + n.T::~decltype(n)(); + // since-cxx11-error@-1 {{expected a class name after '~' to name a destructor}} n.~decltype(n)(); // OK #endif } @@ -141,9 +153,9 @@ namespace dr1758 { // dr1758: 3.7 namespace dr1762 { // dr1762: 14 #if __cplusplus >= 201103L float operator ""_E(const char *); - // expected-error@+2 {{invalid suffix on literal; C++11 requires a space between literal and identifier}} - // expected-warning@+1 {{user-defined literal suffixes not starting with '_' are reserved; no literal will invoke this operator}} float operator ""E(const char *); + // since-cxx11-error@-1 {{invalid suffix on literal; C++11 requires a space between literal and identifier}} + // since-cxx11-warning@-2 {{user-defined literal suffixes not starting with '_' are reserved; no literal will invoke this operator}} #endif } diff --git a/clang/test/CXX/drs/dr18xx.cpp b/clang/test/CXX/drs/dr18xx.cpp index 7ac267373822..fbe67bd0c2f6 100644 --- a/clang/test/CXX/drs/dr18xx.cpp +++ b/clang/test/CXX/drs/dr18xx.cpp @@ -1,14 +1,14 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors - -#if __cplusplus < 201103L -// expected-error@+1 {{variadic macro}} +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected,cxx98 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-17,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-17,since-cxx11,since-cxx14 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,cxx11-17,since-cxx11,since-cxx14 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx14 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx14 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx20,since-cxx11,since-cxx14 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors + +#if __cplusplus == 199711L #define static_assert(...) __extension__ _Static_assert(__VA_ARGS__) +// cxx98-error@-1 {{variadic macros are a C99 feature}} #endif namespace dr1812 { // dr1812: no @@ -16,7 +16,7 @@ namespace dr1812 { // dr1812: no #if __cplusplus >= 201103L template struct A { using B = typename T::C; - // expected-error@-1 {{use 'template' keyword to treat 'C' as a dependent template name}} + // since-cxx11-error@-1 {{use 'template' keyword to treat 'C' as a dependent template name}} }; #endif } // namespace dr1812 @@ -54,15 +54,20 @@ namespace dr1814 { // dr1814: yes namespace dr1815 { // dr1815: no #if __cplusplus >= 201402L // FIXME: needs codegen test - struct A { int &&r = 0; }; // expected-note {{default member init}} - A a = {}; // FIXME expected-warning {{not supported}} - - struct B { int &&r = 0; }; // expected-error {{binds to a temporary}} expected-note {{default member init}} - B b; // expected-note {{here}} + struct A { int &&r = 0; }; // #dr1815-A + A a = {}; + // since-cxx14-warning@-1 {{sorry, lifetime extension of temporary created by aggregate initialization using default member initializer is not supported; lifetime of temporary will end at the end of the full-expression}} FIXME + // since-cxx14-note@#dr1815-A {{initializing field 'r' with default member initializer}} + + struct B { int &&r = 0; }; // #dr1815-B + // since-cxx14-error@-1 {{reference member 'r' binds to a temporary object whose lifetime would be shorter than the lifetime of the constructed object}} + // since-cxx14-note@#dr1815-B {{initializing field 'r' with default member initializer}} + // since-cxx14-note@#dr1815-b {{in implicit default constructor for 'dr1815::B' first required here}} + B b; // #dr1815-b #endif } -namespace dr1821 { // dr1821: yes +namespace dr1821 { // dr1821: 2.9 struct A { template struct B { void f(); @@ -80,9 +85,9 @@ struct A { namespace dr1822 { // dr1822: yes #if __cplusplus >= 201103L - int a; + double a; auto x = [] (int a) { -#pragma clang __debug dump a // CHECK: ParmVarDecl + static_assert(__is_same(decltype(a), int), "should be resolved to lambda parameter"); }; #endif } @@ -98,16 +103,22 @@ namespace dr1837 { // dr1837: 3.3 }; class Outer { - friend auto Other::q() -> decltype(this->p()) *; // expected-error {{invalid use of 'this'}} + friend auto Other::q() -> decltype(this->p()) *; + // since-cxx11-error@-1 {{invalid use of 'this' outside of a non-static member function}} int g(); int f() { extern void f(decltype(this->g()) *); struct Inner { - static_assert(Fishg())>::value, ""); // expected-error {{invalid use of 'this'}} - enum { X = Fishf())>::value }; // expected-error {{invalid use of 'this'}} - struct Inner2 : Fishg())> { }; // expected-error {{invalid use of 'this'}} - friend void f(decltype(this->g()) *); // expected-error {{invalid use of 'this'}} - friend auto Other::q() -> decltype(this->p()) *; // expected-error {{invalid use of 'this'}} + static_assert(Fishg())>::value, ""); + // since-cxx11-error@-1 {{invalid use of 'this' outside of a non-static member function}} + enum { X = Fishf())>::value }; + // since-cxx11-error@-1 {{invalid use of 'this' outside of a non-static member function}} + struct Inner2 : Fishg())> { }; + // since-cxx11-error@-1 {{invalid use of 'this' outside of a non-static member function}} + friend void f(decltype(this->g()) *); + // since-cxx11-error@-1 {{invalid use of 'this' outside of a non-static member function}} + friend auto Other::q() -> decltype(this->p()) *; + // since-cxx11-error@-1 {{invalid use of 'this' outside of a non-static member function}} }; return 0; } @@ -135,19 +146,21 @@ namespace dr1872 { // dr1872: 9 constexpr int x = A().f(); constexpr int y = A().f(); -#if __cplusplus <= 201703L - // expected-error@-2 {{constant expression}} expected-note@-2 {{call to virtual function}} -#else + // cxx11-17-error@-1 {{constexpr variable 'y' must be initialized by a constant expression}} + // cxx11-17-note@-2 {{cannot evaluate call to virtual function in a constant expression in C++ standards before C++20}} +#if __cplusplus >= 202002L static_assert(y == 0); #endif // Note, this is invalid even though it would not use virtual dispatch. constexpr int y2 = A().A::f(); -#if __cplusplus <= 201703L - // expected-error@-2 {{constant expression}} expected-note@-2 {{call to virtual function}} -#else + // cxx11-17-error@-1 {{constexpr variable 'y2' must be initialized by a constant expression}} + // cxx11-17-note@-2 {{cannot evaluate call to virtual function in a constant expression in C++ standards before C++20}} +#if __cplusplus >= 202002L static_assert(y == 0); #endif - constexpr int z = A().f(); // expected-error {{constant expression}} expected-note {{non-literal type}} + constexpr int z = A().f(); + // since-cxx11-error@-1 {{constexpr variable 'z' must be initialized by a constant expression}} + // since-cxx11-note@-2 {{non-literal type 'A' cannot be used in a constant expression}} #endif } @@ -166,33 +179,38 @@ namespace dr1881 { // dr1881: 7 void dr1891() { // dr1891: 4 #if __cplusplus >= 201103L int n; - auto a = []{}; // expected-note 0-4{{}} - auto b = [=]{ return n; }; // expected-note 0-4{{}} + auto a = []{}; // #dr1891-a + auto b = [=]{ return n; }; // #dr1891-b typedef decltype(a) A; typedef decltype(b) B; static_assert(!__has_trivial_constructor(A), ""); -#if __cplusplus > 201703L - // expected-error@-2 {{failed}} -#endif + // since-cxx20-error@-1 {{failed}} static_assert(!__has_trivial_constructor(B), ""); // C++20 allows default construction for non-capturing lambdas (P0624R2). A x; -#if __cplusplus <= 201703L - // expected-error@-2 {{no matching constructor}} -#endif - B y; // expected-error {{no matching constructor}} + // cxx11-17-error@-1 {{no matching constructor for initialization of 'A' (aka '(lambda at}} + // cxx11-17-note@#dr1891-a {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}} + // cxx11-17-note@#dr1891-a {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}} + B y; + // since-cxx11-error@-1 {{no matching constructor for initialization of 'B' (aka '(lambda at}} + // since-cxx11-note@#dr1891-b {{candidate constructor (the implicit copy constructor) not viable: requires 1 argument, but 0 were provided}} + // since-cxx11-note@#dr1891-b {{candidate constructor (the implicit move constructor) not viable: requires 1 argument, but 0 were provided}} // C++20 allows assignment for non-capturing lambdas (P0624R2). a = a; + // cxx11-17-error-re@-1 {{{{object of type '\(lambda at .+\)' cannot be assigned because its copy assignment operator is implicitly deleted}}}} + // cxx11-17-note@#dr1891-a {{lambda expression begins here}} a = static_cast(a); -#if __cplusplus <= 201703L - // expected-error@-3 {{copy assignment operator is implicitly deleted}} - // expected-error@-3 {{copy assignment operator is implicitly deleted}} -#endif - b = b; // expected-error {{copy assignment operator is implicitly deleted}} - b = static_cast(b); // expected-error {{copy assignment operator is implicitly deleted}} + // cxx11-17-error-re@-1 {{{{object of type '\(lambda at .+\)' cannot be assigned because its copy assignment operator is implicitly deleted}}}} + // cxx11-17-note@#dr1891-a {{lambda expression begins here}} + b = b; + // since-cxx11-error-re@-1 {{{{object of type '\(lambda at .+\)' cannot be assigned because its copy assignment operator is implicitly deleted}}}} + // since-cxx11-note@#dr1891-b {{lambda expression begins here}} + b = static_cast(b); + // since-cxx11-error-re@-1 {{{{object of type '\(lambda at .+\)' cannot be assigned because its copy assignment operator is implicitly deleted}}}} + // since-cxx11-note@#dr1891-b {{lambda expression begins here}} #endif } diff --git a/clang/test/CXX/drs/dr19xx.cpp b/clang/test/CXX/drs/dr19xx.cpp index b15be762ecd1..716b1476831e 100644 --- a/clang/test/CXX/drs/dr19xx.cpp +++ b/clang/test/CXX/drs/dr19xx.cpp @@ -1,41 +1,36 @@ -// RUN: %clang_cc1 -std=c++98 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++1z %s -verify -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++98 %s -verify=expected,cxx98-11,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 %s -verify=expected,cxx98-11,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c %s -verify=expected,since-cxx14,since-cxx11 -fexceptions -fcxx-exceptions -pedantic-errors namespace std { struct type_info; } namespace dr1902 { // dr1902: 3.7 struct A {}; struct B { - B(A); -#if __cplusplus >= 201103L - // expected-note@-2 {{candidate}} -#endif - - B() = delete; -#if __cplusplus < 201103L - // expected-error@-2 {{extension}} -#endif - - B(const B&) // expected-note {{deleted here}} -#if __cplusplus >= 201103L - // expected-note@-2 {{candidate}} -#else - // expected-error@+2 {{extension}} -#endif - = delete; - + B(A); // #dr1902-B-A + B() = delete; // #dr1902-B-ctor + // cxx98-error@-1 {{deleted function definitions are a C++11 extension}} + B(const B&) = delete; // #dr1902-B-copy-ctor + // cxx98-error@-1 {{deleted function definitions are a C++11 extension}} operator A(); }; extern B b1; - B b2(b1); // expected-error {{call to deleted}} + B b2(b1); + // expected-error@-1 {{call to deleted constructor of 'B'}} + // expected-note@#dr1902-B-copy-ctor {{'B' has been explicitly marked deleted here}} #if __cplusplus >= 201103L // This is ambiguous, even though calling the B(const B&) constructor would // both directly and indirectly call a deleted function. - B b({}); // expected-error {{ambiguous}} + B b({}); + // since-cxx11-error@-1 {{call to constructor of 'B' is ambiguous}} + // since-cxx11-note@#dr1902-B-A {{candidate constructor}} + // since-cxx11-note@#dr1902-B-copy-ctor {{candidate constructor has been explicitly deleted}} #endif } @@ -64,26 +59,33 @@ namespace dr1903 { } } -namespace dr1909 { // dr1909: yes +namespace dr1909 { // dr1909: 3.7 struct A { - template struct A {}; // expected-error {{member 'A' has the same name as its class}} + template struct A {}; + // expected-error@-1 {{member 'A' has the same name as its class}} }; struct B { - template void B() {} // expected-error {{constructor cannot have a return type}} + template void B() {} + // expected-error@-1 {{constructor cannot have a return type}} }; struct C { - template static int C; // expected-error {{member 'C' has the same name as its class}} expected-error 0-1{{extension}} + template static int C; + // expected-error@-1 {{member 'C' has the same name as its class}} + // cxx98-11-error@-2 {{variable templates are a C++14 extension}} }; struct D { - template using D = int; // expected-error {{member 'D' has the same name as its class}} expected-error 0-1{{extension}} + template using D = int; + // cxx98-error@-1 {{alias declarations are a C++11 extension}} + // expected-error@-2 {{member 'D' has the same name as its class}} }; } -namespace dr1940 { // dr1940: yes +namespace dr1940 { // dr1940: 3.5 #if __cplusplus >= 201103L static union { static_assert(true, ""); // ok - static_assert(false, ""); // expected-error {{static assertion failed}} + static_assert(false, ""); + // since-cxx11-error@-1 {{static assertion failed}} int not_empty; }; #endif @@ -119,18 +121,21 @@ derived d2(42, 9); #endif } -namespace dr1947 { // dr1947: yes +namespace dr1947 { // dr1947: 3.5 #if __cplusplus >= 201402L unsigned o = 0'01; // ok -unsigned b = 0b'01; // expected-error {{invalid digit 'b' in octal constant}} -unsigned x = 0x'01; // expected-error {{invalid suffix 'x'01' on integer constant}} +unsigned b = 0b'01; +// since-cxx14-error@-1 {{invalid digit 'b' in octal constant}} +unsigned x = 0x'01; +// since-cxx14-error@-1 {{invalid suffix 'x'01' on integer constant}} #endif } #if __cplusplus >= 201103L -// dr1948: yes +// dr1948: 3.5 // FIXME: This diagnostic could be improved. -void *operator new(__SIZE_TYPE__) noexcept { return nullptr; } // expected-error{{exception specification in declaration does not match previous declaration}} +void *operator new(__SIZE_TYPE__) noexcept { return nullptr; } +// since-cxx11-error@-1 {{exception specification in declaration does not match previous declaration}} #endif namespace dr1959 { // dr1959: 3.9 @@ -139,22 +144,31 @@ namespace dr1959 { // dr1959: 3.9 struct c; struct a { a() = default; - a(const a &) = delete; // expected-note {{deleted}} + a(const a &) = delete; // #dr1959-copy-ctor a(const b &) = delete; // not inherited - a(c &&) = delete; // expected-note {{not viable}} - template a(T) = delete; // expected-note {{would take its own class type by value}} + a(c &&) = delete; // #dr1959-move-ctor + template a(T) = delete; // #dr1959-temp-ctor }; - struct b : a { // expected-note {{cannot bind}} expected-note {{deleted because}} - using a::a; // expected-note 2{{inherited here}} + struct b : a { // #dr1959-b + using a::a; // #dr1959-using-a }; a x; // FIXME: As a resolution to an open DR against P0136R0, we disallow // use of inherited constructors to construct from a single argument // where the base class is reference-related to the argument type. - b y = x; // expected-error {{no viable conversion}} - b z = z; // expected-error {{deleted}} + b y = x; + // since-cxx11-error@-1 {{no viable conversion from 'a' to 'b'}} + // since-cxx11-note@#dr1959-move-ctor {{candidate inherited constructor not viable: no known conversion from 'a' to 'c &&' for 1st argument}} + // since-cxx11-note@#dr1959-using-a {{constructor from base class 'a' inherited here}} + // since-cxx11-note@#dr1959-b {{candidate constructor (the implicit copy constructor) not viable: cannot bind base class object of type 'a' to derived class reference 'const b &' for 1st argument}} + // since-cxx11-note@#dr1959-temp-ctor {{candidate template ignored: instantiation would take its own class type by value}} + // since-cxx11-note@#dr1959-using-a {{constructor from base class 'a' inherited here}} + b z = z; + // since-cxx11-error@-1 {{call to implicitly-deleted copy constructor of 'b'}} + // since-cxx11-note@#dr1959-b {{copy constructor of 'b' is implicitly deleted because base class 'a' has a deleted copy constructor}} + // since-cxx11-note@#dr1959-copy-ctor {{'a' has been explicitly marked deleted here}} struct c : a { using a::a; @@ -191,16 +205,28 @@ using A::g; namespace dr1966 { // dr1966: 11 #if __cplusplus >= 201103L struct A { - enum E : int {1}; // expected-error {{expected identifier}} (not bit-field) + enum E : int {1}; + // since-cxx11-error@-1 {{expected identifier}} (not bit-field) }; - auto *p1 = new enum E : int; // expected-error {{only permitted as a standalone declaration}} - auto *p2 = new enum F : int {}; // expected-error {{only permitted as a standalone declaration}} - auto *p3 = true ? new enum G : int {}; // expected-error {{forward reference}} expected-error {{incomplete}} expected-note {{declaration}} - auto h() -> enum E : int {}; // expected-error {{only permitted as a standalone declaration}} - - enum X : enum Y : int {} {}; // expected-error {{cannot be defined in a type specifier}} + auto *p1 = new enum E : int; + // since-cxx11-error@-1 {{non-defining declaration of enumeration with a fixed underlying type is only permitted as a standalone declaration}} + auto *p2 = new enum F : int {}; + // since-cxx11-error@-1 {{non-defining declaration of enumeration with a fixed underlying type is only permitted as a standalone declaration}} + auto *p3 = true ? new enum G : int {}; + // since-cxx11-error@-1 {{ISO C++ forbids forward references to 'enum' types}} + // since-cxx11-error@-2 {{allocation of incomplete type 'enum G'}} + // since-cxx11-note@-3 {{forward declaration of 'dr1966::G'}} + auto h() -> enum E : int {}; + // since-cxx11-error@-1 {{non-defining declaration of enumeration with a fixed underlying type is only permitted as a standalone declaration}} + + enum X : enum Y : int {} {}; + // since-cxx11-error@-1 {{'dr1966::Y' cannot be defined in a type specifier}} struct Q { - enum X : enum Y : int {} {}; // expected-error +{{}} + // FIXME: can we emit something nicer than that? + enum X : enum Y : int {} {}; + // since-cxx11-error@-1 {{non-defining declaration of enumeration with a fixed underlying type is only permitted as a standalone declaration; missing list of enumerators?}} + // since-cxx11-error@-2 {{non-integral type 'enum Y' is an invalid underlying type}} + // since-cxx11-error@-3 {{anonymous bit-field cannot have a default member initializer}} }; #endif } diff --git a/clang/test/CXX/drs/dr20xx.cpp b/clang/test/CXX/drs/dr20xx.cpp index 4f81b0b413d4..60ee7684440f 100644 --- a/clang/test/CXX/drs/dr20xx.cpp +++ b/clang/test/CXX/drs/dr20xx.cpp @@ -1,13 +1,14 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors \ -// RUN: -Wno-variadic-macros -Wno-c11-extensions -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -fcxx-exceptions -pedantic-errors - -#if __cplusplus < 201103L -#define static_assert(...) _Static_assert(__VA_ARGS__) +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected,cxx98 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,cxx11 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx14 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11,since-cxx14,since-cxx20 -fexceptions -fcxx-exceptions -pedantic-errors + +#if __cplusplus == 199711L +#define static_assert(...) __extension__ _Static_assert(__VA_ARGS__) +// cxx98-error@-1 {{variadic macros are a C99 feature}} #endif namespace dr2007 { // dr2007: 3.4 @@ -15,8 +16,12 @@ template struct A { typename T::error e; }; template struct B { }; B > b1; B > b2 = b1; -int a = b2[0]; // expected-error {{does not provide a subscript operator}} -int b = __builtin_addressof(b2)->foo; // expected-error {{no member}} +int a = b2[0]; +// cxx98-error@-1 {{type 'B >' does not provide a subscript operator}} +// since-cxx11-error@-2 {{type 'B>' does not provide a subscript operator}} +int b = __builtin_addressof(b2)->foo; +// cxx98-error@-1 {{no member named 'foo' in 'dr2007::B >'}} +// since-cxx11-error@-2 {{no member named 'foo' in 'dr2007::B>'}} } // dr2009: na @@ -24,45 +29,69 @@ int b = __builtin_addressof(b2)->foo; // expected-error {{no member}} namespace dr2026 { // dr2026: 11 template struct X {}; - const int a = a + 1; // expected-warning {{uninitialized}} expected-note {{here}} expected-note 0-1{{outside its lifetime}} - X xa; // expected-error {{constant expression}} expected-note {{initializer of 'a'}} + const int a = a + 1; // #dr2026-a + // expected-warning@-1 {{variable 'a' is uninitialized when used within its own initialization}} + X xa; // #dr2026-xa + // cxx98-error@-1 {{non-type template argument of type 'int' is not an integral constant expression}} + // cxx98-note@-2 {{initializer of 'a' is not a constant expression}} + // cxx98-note@#dr2026-a {{declared here}} + // since-cxx11-error@#dr2026-xa {{non-type template argument is not a constant expression}} + // since-cxx11-note@#dr2026-xa {{initializer of 'a' is not a constant expression}} + // since-cxx11-note@#dr2026-a {{declared here}} #if __cplusplus >= 201103L - constexpr int b = b; // expected-error {{constant expression}} expected-note {{outside its lifetime}} - [[clang::require_constant_initialization]] int c = c; // expected-error {{constant initializer}} expected-note {{attribute}} -#if __cplusplus == 201103L - // expected-note@-2 {{read of non-const variable}} expected-note@-2 {{declared here}} -#else - // expected-note@-4 {{outside its lifetime}} -#endif + constexpr int b = b; + // since-cxx11-error@-1 {{constexpr variable 'b' must be initialized by a constant expression}} + // since-cxx11-note@-2 {{read of object outside its lifetime is not allowed in a constant expression}} + [[clang::require_constant_initialization]] int c = c; + // since-cxx11-error@-1 {{variable does not have a constant initializer}} + // since-cxx11-note@-2 {{required by 'require_constant_initialization' attribute here}} + // cxx11-note@-3 {{read of non-const variable 'c' is not allowed in a constant expression}} + // cxx11-note@-4 {{declared here}} + // since-cxx14-note@-5 {{read of object outside its lifetime is not allowed in a constant expression}} #endif -#if __cplusplus > 201703L - constinit int d = d; // expected-error {{constant initializer}} expected-note {{outside its lifetime}} expected-note {{'constinit'}} +#if __cplusplus >= 202002L + constinit int d = d; + // since-cxx20-error@-1 {{variable does not have a constant initializer}} + // since-cxx20-note@-2 {{required by 'constinit' specifier here}} + // since-cxx20-note@-3 {{read of object outside its lifetime is not allowed in a constant expression}} #endif void f() { - static const int e = e + 1; // expected-warning {{suspicious}} expected-note {{here}} expected-note 0-1{{outside its lifetime}} - X xe; // expected-error {{constant expression}} expected-note {{initializer of 'e'}} + static const int e = e + 1; // #dr2026-e + // expected-warning@-1 {{static variable 'e' is suspiciously used within its own initialization}} + X xe; // #dr2026-xe + // cxx98-error@-1 {{non-type template argument of type 'int' is not an integral constant expression}} + // cxx98-note@-2 {{initializer of 'e' is not a constant expression}} + // cxx98-note@#dr2026-e {{declared here}} + // since-cxx11-error@#dr2026-xe {{non-type template argument is not a constant expression}} + // since-cxx11-note@#dr2026-xe {{initializer of 'e' is not a constant expression}} + // since-cxx11-note@#dr2026-e {{declared here}} #if __cplusplus >= 201103L - static constexpr int f = f; // expected-error {{constant expression}} expected-note {{outside its lifetime}} - [[clang::require_constant_initialization]] static int g = g; // expected-error {{constant initializer}} expected-note {{attribute}} -#if __cplusplus == 201103L - // expected-note@-2 {{read of non-const variable}} expected-note@-2 {{declared here}} -#else - // expected-note@-4 {{outside its lifetime}} -#endif + static constexpr int f = f; + // since-cxx11-error@-1 {{constexpr variable 'f' must be initialized by a constant expression}} + // since-cxx11-note@-2 {{read of object outside its lifetime is not allowed in a constant expression}} + [[clang::require_constant_initialization]] static int g = g; + // since-cxx11-error@-1 {{variable does not have a constant initializer}} + // since-cxx11-note@-2 {{required by 'require_constant_initialization' attribute here}} + // cxx11-note@-3 {{read of non-const variable 'g' is not allowed in a constant expression}} + // cxx11-note@-4 {{declared here}} + // since-cxx14-note@-5 {{read of object outside its lifetime is not allowed in a constant expression}} #endif -#if __cplusplus > 201703L - static constinit int h = h; // expected-error {{constant initializer}} expected-note {{outside its lifetime}} expected-note {{'constinit'}} +#if __cplusplus >= 202002L + static constinit int h = h; + // since-cxx20-error@-1 {{variable does not have a constant initializer}} + // since-cxx20-note@-2 {{required by 'constinit' specifier here}} + // since-cxx20-note@-3 {{read of object outside its lifetime is not allowed in a constant expression}} #endif } } namespace dr2049 { // dr2049: 18 drafting -#if __cplusplus > 202002L +#if __cplusplus >= 202302L template struct X {}; X<> a; X b; @@ -120,8 +149,8 @@ namespace dr2076 { // dr2076: 13 operator string_view() const; }; - void foo(const string &); // expected-note {{cannot convert initializer list}} - void bar(string_view); // expected-note 2{{cannot convert initializer list}} + void foo(const string &); // #dr2076-foo + void bar(string_view); // #dr2076-bar void func(const string &arg) { // An argument in one set of braces is subject to user-defined conversions; @@ -130,11 +159,17 @@ namespace dr2076 { // dr2076: 13 foo(arg); foo({arg}); foo({{arg}}); - foo({{{arg}}}); // expected-error {{no matching function}} + foo({{{arg}}}); + // since-cxx11-error@-1 {{no matching function}} + // since-cxx11-note@#dr2076-foo {{cannot convert initializer list}} bar(arg); bar({arg}); - bar({{arg}}); // expected-error {{no matching function}} - bar({{{arg}}}); // expected-error {{no matching function}} + bar({{arg}}); + // since-cxx11-error@-1 {{no matching function}} + // since-cxx11-note@#dr2076-bar {{cannot convert initializer list}} + bar({{{arg}}}); + // since-cxx11-error@-1 {{no matching function}} + // since-cxx11-note@#dr2076-bar {{cannot convert initializer list}} } #endif } @@ -172,18 +207,20 @@ namespace dr2083 { // dr2083: partial // treatment in C++11 onwards. We continue to apply that even after DR2083. void ref_to_non_const() { int c; - const int &ra = a; // expected-note 0-1{{here}} - int &rb = b; // expected-note 0-1{{here}} - int &rc = c; // expected-note {{here}} + const int &ra = a; // #dr2083-ra + int &rb = b; // #dr2083-rb + int &rc = c; // #dr2083-rc struct A { int f() { int a = ra; + // cxx98-error@-1 {{reference to local variable 'ra' declared in enclosing function 'dr2083::ref_to_non_const'}} + // cxx98-note@#dr2083-ra {{'ra' declared here}} int b = rb; -#if __cplusplus < 201103L - // expected-error@-3 {{in enclosing function}} - // expected-error@-3 {{in enclosing function}} -#endif - int c = rc; // expected-error {{in enclosing function}} + // cxx98-error@-1 {{reference to local variable 'rb' declared in enclosing function 'dr2083::ref_to_non_const'}} + // cxx98-note@#dr2083-rb {{'rb' declared here}} + int c = rc; + // expected-error@-1 {{reference to local variable 'rc' declared in enclosing function 'dr2083::ref_to_non_const'}} + // expected-note@#dr2083-rc {{'rc' declared here}} return a + b + c; } }; @@ -207,18 +244,24 @@ namespace dr2083 { // dr2083: partial constexpr NoMut1 nm1 = {1, 2}; constexpr NoMut2 nm2 = {1, 2}; constexpr NoMut3 nm3 = {1, 2}; - constexpr Mut1 m1 = {1, 2}; // expected-note {{declared here}} - constexpr Mut2 m2 = {1, 2}; // expected-note {{declared here}} - constexpr Mut3 m3 = {1, 2}; // expected-note {{declared here}} + constexpr Mut1 m1 = {1, 2}; // #dr2083-m1 + constexpr Mut2 m2 = {1, 2}; // #dr2083-m2 + constexpr Mut3 m3 = {1, 2}; // #dr2083-m3 struct A { void f() { static_assert(nm1.a == 1, ""); static_assert(nm2.m.a == 1, ""); static_assert(nm3.a == 1, ""); // Can't even access a non-mutable member of a variable containing mutable fields. - static_assert(m1.a == 1, ""); // expected-error {{enclosing function}} - static_assert(m2.m.a == 1, ""); // expected-error {{enclosing function}} - static_assert(m3.a == 1, ""); // expected-error {{enclosing function}} + static_assert(m1.a == 1, ""); + // since-cxx11-error@-1 {{reference to local variable 'm1' declared in enclosing function 'dr2083::mutable_subobjects'}} + // since-cxx11-note@#dr2083-m1 {{'m1' declared here}} + static_assert(m2.m.a == 1, ""); + // since-cxx11-error@-1 {{reference to local variable 'm2' declared in enclosing function 'dr2083::mutable_subobjects'}} + // since-cxx11-note@#dr2083-m2 {{'m2' declared here}} + static_assert(m3.a == 1, ""); + // since-cxx11-error@-1 {{reference to local variable 'm3' declared in enclosing function 'dr2083::mutable_subobjects'}} + // since-cxx11-note@#dr2083-m3 {{'m3' declared here}} } }; } @@ -231,14 +274,16 @@ namespace dr2083 { // dr2083: partial #if __cplusplus >= 201103L constexpr #endif - A a = {}; // expected-note {{here}} + A a = {}; // #dr2083-a struct B { void f() { ellipsis(n); // Even though this is technically modelled as an lvalue-to-rvalue // conversion, it calls a constructor and binds 'a' to a reference, so // it results in an odr-use. - ellipsis(a); // expected-error {{enclosing function}} + ellipsis(a); + // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr2083::ellipsis'}} + // expected-note@#dr2083-a {{'a' declared here}} } }; } @@ -246,7 +291,7 @@ namespace dr2083 { // dr2083: partial #if __cplusplus >= 201103L void volatile_lval() { struct A { int n; }; - constexpr A a = {0}; // expected-note {{here}} + constexpr A a = {0}; // #dr2083-a2 struct B { void f() { // An lvalue-to-rvalue conversion of a volatile lvalue always results @@ -254,7 +299,9 @@ namespace dr2083 { // dr2083: partial int A::*p = &A::n; int x = a.*p; volatile int A::*q = p; - int y = a.*q; // expected-error {{enclosing function}} + int y = a.*q; + // since-cxx11-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr2083::volatile_lval'}} + // since-cxx11-note@#dr2083-a2 {{'a' declared here}} } }; } @@ -262,32 +309,45 @@ namespace dr2083 { // dr2083: partial void discarded_lval() { struct A { int x; mutable int y; volatile int z; }; - A a; // expected-note 1+{{here}} - int &r = a.x; // expected-note {{here}} + A a; // #dr2083-a-3 + int &r = a.x; // #dr2083-r struct B { void f() { - a.x; // expected-warning {{unused}} - a.*&A::x; // expected-warning {{unused}} - true ? a.x : a.y; // expected-warning {{unused}} + // FIXME: We emit more errors than we should be. They are explictly marked below. + a.x; + // expected-warning@-1 {{expression result unused}} + // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME + // expected-note@#dr2083-a-3 {{'a' declared here}} + a.*&A::x; + // expected-warning@-1 {{expression result unused}} + // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME + // expected-note@#dr2083-a-3 {{'a' declared here}} + true ? a.x : a.y; // #dr2083-ternary + // expected-warning@-1 {{expression result unused}} + // expected-error@#dr2083-ternary {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME + // expected-note@#dr2083-a-3 {{'a' declared here}} + // expected-error@#dr2083-ternary {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME + // expected-note@#dr2083-a-3 {{'a' declared here}} (void)a.x; - a.x, discarded_lval(); // expected-warning {{left operand of comma operator has no effect}} -#if 1 // FIXME: These errors are all incorrect; the above code is valid. - // expected-error@-6 {{enclosing function}} - // expected-error@-6 {{enclosing function}} - // expected-error@-6 2{{enclosing function}} - // expected-error@-6 {{enclosing function}} - // expected-error@-6 {{enclosing function}} -#endif + // expected-error@-1 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME + // expected-note@#dr2083-a-3 {{'a' declared here}} + a.x, discarded_lval(); + // expected-warning@-1 {{left operand of comma operator has no effect}} + // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} FIXME + // expected-note@#dr2083-a-3 {{'a' declared here}} // 'volatile' qualifier triggers an lvalue-to-rvalue conversion. - a.z; // expected-error {{enclosing function}} -#if __cplusplus < 201103L - // expected-warning@-2 {{assign into a variable}} -#endif + a.z; + // cxx98-warning@-1 {{expression result unused; assign into a variable to force a volatile load}} + // expected-error@-2 {{reference to local variable 'a' declared in enclosing function 'dr2083::discarded_lval'}} + // expected-note@#dr2083-a-3 {{'a' declared here}} // References always get "loaded" to determine what they reference, // even if the result is discarded. - r; // expected-error {{enclosing function}} expected-warning {{unused}} + r; + // expected-warning@-1 {{expression result unused}} + // expected-error@-2 {{reference to local variable 'r' declared in enclosing function 'dr2083::discarded_lval'}} + // expected-note@#dr2083-r {{'r' declared here}} } }; } @@ -295,12 +355,11 @@ namespace dr2083 { // dr2083: partial namespace dr_example_1 { extern int globx; int main() { - const int &x = globx; + const int &x = globx; // #dr2083-x struct A { -#if __cplusplus < 201103L - // expected-error@+2 {{enclosing function}} expected-note@-3 {{here}} -#endif const int *foo() { return &x; } + // cxx98-error@-1 {{reference to local variable 'x' declared in enclosing function 'dr2083::dr_example_1::main'}} + // cxx98-note@#dr2083-x {{'x' declared here}} } a; return *a.foo(); } diff --git a/clang/test/CXX/drs/dr21xx.cpp b/clang/test/CXX/drs/dr21xx.cpp index a1b8fe3f2a9b..a7e50df3f374 100644 --- a/clang/test/CXX/drs/dr21xx.cpp +++ b/clang/test/CXX/drs/dr21xx.cpp @@ -1,13 +1,14 @@ -// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors -// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors - -#if __cplusplus < 201103L -// expected-error@+1 {{variadic macro}} +// RUN: %clang_cc1 -std=c++98 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,cxx98 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++11 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++14 -triple x86_64-unknown-unknown %s -verify=expected,cxx98-14,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++17 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++20 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++23 -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors +// RUN: %clang_cc1 -std=c++2c -triple x86_64-unknown-unknown %s -verify=expected,since-cxx11 -fexceptions -Wno-deprecated-builtins -fcxx-exceptions -pedantic-errors + +#if __cplusplus == 199711L #define static_assert(...) __extension__ _Static_assert(__VA_ARGS__) +// cxx98-error@-1 {{variadic macros are a C99 feature}} #endif namespace dr2100 { // dr2100: 12 @@ -18,15 +19,14 @@ namespace dr2100 { // dr2100: 12 return X<&n>::n; // ok, value-dependent } int g() { - static const int n = 2; + static const int n = 2; // #dr2100-n return X<&n>::n; // ok, value-dependent -#if __cplusplus < 201702L - // expected-error@-2 {{does not have linkage}} expected-note@-3 {{here}} -#endif + // cxx98-14-error@-1 {{non-type template argument refers to object 'n' that does not have linkage}} + // cxx98-14-note@#dr2100-n {{non-type template argument refers to object here}} } }; template struct X